testing/web-platform/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the web-platform-tests test runner with mach.
   6
   7 import os
   8 import sys
   9
  10 from mach.decorators import Command
  11 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
  12 from mozbuild.base import MachCommandConditions as conditions
  13 from mozbuild.base import MozbuildObject
  14 from six import iteritems
  15
  16 here = os.path.abspath(os.path.dirname(__file__))
  17 INTEROP_REQUIREMENTS_PATH = os.path.join(here, "interop_requirements.txt")
  18
  19
  20 class WebPlatformTestsRunnerSetup(MozbuildObject):
  21     default_log_type = "mach"
  22
  23     def __init__(self, *args, **kwargs):
  24         super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
  25         self._here = os.path.join(self.topsrcdir, "testing", "web-platform")
  26         kwargs["tests_root"] = os.path.join(self._here, "tests")
  27         sys.path.insert(0, kwargs["tests_root"])
  28         build_path = os.path.join(self.topobjdir, "build")
  29         if build_path not in sys.path:
  30             sys.path.append(build_path)
  31
  32     def kwargs_common(self, kwargs):
  33         """Setup kwargs relevant for all browser products"""
  34
  35         tests_src_path = os.path.join(self._here, "tests")
  36
  37         if (
  38             kwargs["product"] in {"firefox", "firefox_android"}
  39             and kwargs["specialpowers_path"] is None
  40         ):
  41             kwargs["specialpowers_path"] = os.path.join(
  42                 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi"
  43             )
  44
  45         if kwargs["product"] == "firefox_android":
  46             # package_name may be different in the future
  47             package_name = kwargs["package_name"]
  48             if not package_name:
  49                 kwargs[
  50                     "package_name"
  51                 ] = package_name = "org.mozilla.geckoview.test_runner"
  52
  53             # Note that this import may fail in non-firefox-for-android trees
  54             from mozrunner.devices.android_device import (
  55                 InstallIntent,
  56                 get_adb_path,
  57                 verify_android_device,
  58             )
  59
  60             kwargs["adb_binary"] = get_adb_path(self)
  61             install = (
  62                 InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES
  63             )
  64             verify_android_device(
  65                 self, install=install, verbose=False, xre=True, app=package_name
  66             )
  67
  68             if kwargs["certutil_binary"] is None:
  69                 kwargs["certutil_binary"] = os.path.join(
  70                     os.environ.get("MOZ_HOST_BIN"), "certutil"
  71                 )
  72
  73             if kwargs["install_fonts"] is None:
  74                 kwargs["install_fonts"] = True
  75
  76             if not kwargs["device_serial"]:
  77                 kwargs["device_serial"] = ["emulator-5554"]
  78
  79         if kwargs["config"] is None:
  80             kwargs["config"] = os.path.join(
  81                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
  82             )
  83
  84         if (
  85             kwargs["exclude"] is None
  86             and kwargs["include"] is None
  87             and not sys.platform.startswith("linux")
  88         ):
  89             kwargs["exclude"] = ["css"]
  90
  91         if kwargs["ssl_type"] in (None, "pregenerated"):
  92             cert_root = os.path.join(tests_src_path, "tools", "certs")
  93             if kwargs["ca_cert_path"] is None:
  94                 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
  95
  96             if kwargs["host_key_path"] is None:
  97                 kwargs["host_key_path"] = os.path.join(
  98                     cert_root, "web-platform.test.key"
  99                 )
 100
 101             if kwargs["host_cert_path"] is None:
 102                 kwargs["host_cert_path"] = os.path.join(
 103                     cert_root, "web-platform.test.pem"
 104                 )
 105
 106         if kwargs["reftest_screenshot"] is None:
 107             kwargs["reftest_screenshot"] = "fail"
 108
 109         kwargs["capture_stdio"] = True
 110
 111         return kwargs
 112
 113     def kwargs_firefox(self, kwargs):
 114         """Setup kwargs specific to running Firefox and other gecko browsers"""
 115         import mozinfo
 116         from wptrunner import wptcommandline
 117
 118         kwargs = self.kwargs_common(kwargs)
 119
 120         if kwargs["binary"] is None:
 121             kwargs["binary"] = self.get_binary_path()
 122
 123         if kwargs["certutil_binary"] is None:
 124             kwargs["certutil_binary"] = self.get_binary_path("certutil")
 125
 126         if kwargs["webdriver_binary"] is None:
 127             kwargs["webdriver_binary"] = self.get_binary_path(
 128                 "geckodriver", validate_exists=False
 129             )
 130
 131         if kwargs["install_fonts"] is None:
 132             kwargs["install_fonts"] = True
 133
 134         if (
 135             kwargs["install_fonts"]
 136             and mozinfo.info["os"] == "win"
 137             and mozinfo.info["os_version"] == "6.1"
 138         ):
 139             # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
 140             self.setup_fonts_firefox()
 141             kwargs["install_fonts"] = False
 142
 143         if kwargs["preload_browser"] is None:
 144             kwargs["preload_browser"] = False
 145
 146         if kwargs["prefs_root"] is None:
 147             kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles")
 148
 149         if kwargs["stackfix_dir"] is None:
 150             kwargs["stackfix_dir"] = self.bindir
 151
 152         kwargs = wptcommandline.check_args(kwargs)
 153
 154         return kwargs
 155
 156     def kwargs_wptrun(self, kwargs):
 157         """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
 158         from tools.wpt import run, virtualenv
 159
 160         kwargs = self.kwargs_common(kwargs)
 161
 162         # Our existing kwargs corresponds to the wptrunner command line arguments.
 163         # `wpt run` extends this with some additional arguments that are consumed by
 164         # the frontend. Copy over the default values of these extra arguments so they
 165         # are present when we call into that frontend.
 166         run_parser = run.create_parser()
 167         run_kwargs = run_parser.parse_args([kwargs["product"], kwargs["test_list"]])
 168
 169         for key, value in vars(run_kwargs).items():
 170             if key not in kwargs:
 171                 kwargs[key] = value
 172
 173         # Install the deps
 174         # We do this explicitly to avoid calling pip with options that aren't
 175         # supported in the in-tree version
 176         wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
 177         browser_cls = run.product_setup[kwargs["product"]].browser_cls
 178         requirements = ["requirements.txt"]
 179         if hasattr(browser_cls, "requirements"):
 180             requirements.append(browser_cls.requirements)
 181
 182         for filename in requirements:
 183             path = os.path.join(wptrunner_path, filename)
 184             if os.path.exists(path):
 185                 self.virtualenv_manager.install_pip_requirements(
 186                     path, require_hashes=False
 187                 )
 188
 189         venv = virtualenv.Virtualenv(
 190             self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
 191         )
 192         try:
 193             kwargs = run.setup_wptrunner(venv, **kwargs)
 194         except run.WptrunError as e:
 195             print(e, file=sys.stderr)
 196             sys.exit(1)
 197
 198         # This is kind of a hack; override the metadata paths so we don't use
 199         # gecko metadata for non-gecko products
 200         for key, value in list(iteritems(kwargs["test_paths"])):
 201             meta_suffix = key.strip("/")
 202             meta_dir = os.path.join(
 203                 self._here, "products", kwargs["product"].name, meta_suffix
 204             )
 205             value["metadata_path"] = meta_dir
 206             if not os.path.exists(meta_dir):
 207                 os.makedirs(meta_dir)
 208         return kwargs
 209
 210     def setup_fonts_firefox(self):
 211         # Ensure the Ahem font is available
 212         if not sys.platform.startswith("darwin"):
 213             font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
 214         else:
 215             font_path = os.path.join(
 216                 os.path.dirname(self.get_binary_path()),
 217                 os.pardir,
 218                 "Resources",
 219                 "res",
 220                 "fonts",
 221             )
 222         ahem_src = os.path.join(
 223             self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
 224         )
 225         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 226         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
 227             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
 228                 dest.write(src.read())
 229
 230
 231 class WebPlatformTestsServeRunner(MozbuildObject):
 232     def run(self, **kwargs):
 233         sys.path.insert(
 234             0,
 235             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 236         )
 237         import logging
 238
 239         import manifestupdate
 240         from serve import serve
 241         from wptrunner import wptcommandline
 242
 243         logger = logging.getLogger("web-platform-tests")
 244
 245         src_root = self.topsrcdir
 246         obj_root = self.topobjdir
 247         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 248
 249         config_path = manifestupdate.generate_config(
 250             logger,
 251             src_root,
 252             src_wpt_dir,
 253             os.path.join(obj_root, "_tests", "web-platform"),
 254             False,
 255         )
 256
 257         test_paths = wptcommandline.get_test_paths(
 258             wptcommandline.config.read(config_path)
 259         )
 260
 261         def get_route_builder(*args, **kwargs):
 262             route_builder = serve.get_route_builder(*args, **kwargs)
 263
 264             for url_base, paths in iteritems(test_paths):
 265                 if url_base != "/":
 266                     route_builder.add_mount_point(url_base, paths["tests_path"])
 267
 268             return route_builder
 269
 270         return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1
 271
 272
 273 class WebPlatformTestsUpdater(MozbuildObject):
 274     """Update web platform tests."""
 275
 276     def setup_logging(self, **kwargs):
 277         import update
 278
 279         return update.setup_logging(kwargs, {"mach": sys.stdout})
 280
 281     def update_manifest(self, logger, **kwargs):
 282         import manifestupdate
 283
 284         return manifestupdate.run(
 285             logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs
 286         )
 287
 288     def run_update(self, logger, **kwargs):
 289         import update
 290         from update import updatecommandline
 291
 292         self.update_manifest(logger, **kwargs)
 293
 294         if kwargs["config"] is None:
 295             kwargs["config"] = os.path.join(
 296                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
 297             )
 298         if kwargs["product"] is None:
 299             kwargs["product"] = "firefox"
 300
 301         kwargs["store_state"] = False
 302
 303         kwargs = updatecommandline.check_args(kwargs)
 304
 305         try:
 306             update.run_update(logger, **kwargs)
 307         except Exception:
 308             import traceback
 309
 310             traceback.print_exc()
 311
 312
 313 class WebPlatformTestsUnittestRunner(MozbuildObject):
 314     def run(self, **kwargs):
 315         import unittestrunner
 316
 317         return unittestrunner.run(self.topsrcdir, **kwargs)
 318
 319
 320 class WebPlatformTestsTestPathsRunner(MozbuildObject):
 321     """Update web platform tests."""
 322
 323     def run(self, **kwargs):
 324         sys.path.insert(
 325             0,
 326             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 327         )
 328         import logging
 329
 330         import manifestupdate
 331         from manifest import testpaths
 332         from wptrunner import wptcommandline
 333
 334         logger = logging.getLogger("web-platform-tests")
 335
 336         src_root = self.topsrcdir
 337         obj_root = self.topobjdir
 338         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 339
 340         config_path = manifestupdate.generate_config(
 341             logger,
 342             src_root,
 343             src_wpt_dir,
 344             os.path.join(obj_root, "_tests", "web-platform"),
 345             False,
 346         )
 347
 348         test_paths = wptcommandline.get_test_paths(
 349             wptcommandline.config.read(config_path)
 350         )
 351         results = {}
 352         for url_base, paths in iteritems(test_paths):
 353             if "manifest_path" not in paths:
 354                 paths["manifest_path"] = os.path.join(
 355                     paths["metadata_path"], "MANIFEST.json"
 356                 )
 357             results.update(
 358                 testpaths.get_paths(
 359                     path=paths["manifest_path"],
 360                     src_root=src_root,
 361                     tests_root=paths["tests_path"],
 362                     update=kwargs["update"],
 363                     rebuild=kwargs["rebuild"],
 364                     url_base=url_base,
 365                     cache_root=kwargs["cache_root"],
 366                     test_ids=kwargs["test_ids"],
 367                 )
 368             )
 369         testpaths.write_output(results, kwargs["json"])
 370         return True
 371
 372
 373 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject):
 374     def run(self, **kwargs):
 375         import fissionregressions
 376         import mozlog
 377
 378         src_root = self.topsrcdir
 379         obj_root = self.topobjdir
 380         logger = mozlog.structuredlog.StructuredLogger("web-platform-tests")
 381
 382         try:
 383             return fissionregressions.run(logger, src_root, obj_root, **kwargs)
 384         except Exception:
 385             import pdb
 386             import traceback
 387
 388             traceback.print_exc()
 389             pdb.post_mortem()
 390
 391
 392 def create_parser_update():
 393     from update import updatecommandline
 394
 395     return updatecommandline.create_parser()
 396
 397
 398 def create_parser_manifest_update():
 399     import manifestupdate
 400
 401     return manifestupdate.create_parser()
 402
 403
 404 def create_parser_metadata_summary():
 405     import metasummary
 406
 407     return metasummary.create_parser()
 408
 409
 410 def create_parser_metadata_merge():
 411     import metamerge
 412
 413     return metamerge.get_parser()
 414
 415
 416 def create_parser_serve():
 417     sys.path.insert(
 418         0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools"))
 419     )
 420     import serve
 421
 422     return serve.serve.get_parser()
 423
 424
 425 def create_parser_unittest():
 426     import unittestrunner
 427
 428     return unittestrunner.get_parser()
 429
 430
 431 def create_parser_fission_regressions():
 432     import fissionregressions
 433
 434     return fissionregressions.get_parser()
 435
 436
 437 def create_parser_fetch_logs():
 438     import interop
 439
 440     return interop.get_parser_fetch_logs()
 441
 442
 443 def create_parser_interop_score():
 444     import interop
 445
 446     return interop.get_parser_interop_score()
 447
 448
 449 def create_parser_testpaths():
 450     import argparse
 451
 452     from mach.util import get_state_dir
 453
 454     parser = argparse.ArgumentParser()
 455     parser.add_argument(
 456         "--no-update",
 457         dest="update",
 458         action="store_false",
 459         default=True,
 460         help="Don't update manifest before continuing",
 461     )
 462     parser.add_argument(
 463         "-r",
 464         "--rebuild",
 465         action="store_true",
 466         default=False,
 467         help="Force a full rebuild of the manifest.",
 468     )
 469     parser.add_argument(
 470         "--cache-root",
 471         action="store",
 472         default=os.path.join(get_state_dir(), "cache", "wpt"),
 473         help="Path in which to store any caches (default <tests_root>/.wptcache/)",
 474     )
 475     parser.add_argument(
 476         "test_ids", action="store", nargs="+", help="Test ids for which to get paths"
 477     )
 478     parser.add_argument(
 479         "--json", action="store_true", default=False, help="Output as JSON"
 480     )
 481     return parser
 482
 483
 484 @Command(
 485     "web-platform-tests",
 486     category="testing",
 487     conditions=[conditions.is_firefox_or_android],
 488     description="Run web-platform-tests.",
 489     parser=create_parser_wpt,
 490     virtualenv_name="wpt",
 491 )
 492 def run_web_platform_tests(command_context, **params):
 493     if params["product"] is None:
 494         if conditions.is_android(command_context):
 495             params["product"] = "firefox_android"
 496         else:
 497             params["product"] = "firefox"
 498     if "test_objects" in params:
 499         include = []
 500         test_types = set()
 501         for item in params["test_objects"]:
 502             include.append(item["name"])
 503             test_types.add(item.get("subsuite"))
 504         if None not in test_types:
 505             params["test_types"] = list(test_types)
 506         params["include"] = include
 507         del params["test_objects"]
 508         # subsuite coming from `mach test` means something more like `test type`, so remove that argument
 509         if "subsuite" in params:
 510             del params["subsuite"]
 511     if params.get("debugger", None):
 512         import mozdebug
 513
 514         if not mozdebug.get_debugger_info(params.get("debugger")):
 515             sys.exit(1)
 516
 517     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 518     wpt_setup._mach_context = command_context._mach_context
 519     wpt_setup._virtualenv_name = command_context._virtualenv_name
 520     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 521
 522     logger = wpt_runner.setup_logging(**params)
 523     # wptrunner already handles setting any log parameter from
 524     # mach test to the logger, so it's OK to remove that argument now
 525     if "log" in params:
 526         del params["log"]
 527
 528     if (
 529         conditions.is_android(command_context)
 530         and params["product"] != "firefox_android"
 531     ):
 532         logger.warning("Must specify --product=firefox_android in Android environment.")
 533
 534     return wpt_runner.run(logger, **params)
 535
 536
 537 @Command(
 538     "wpt",
 539     category="testing",
 540     conditions=[conditions.is_firefox_or_android],
 541     description="Run web-platform-tests.",
 542     parser=create_parser_wpt,
 543     virtualenv_name="wpt",
 544 )
 545 def run_wpt(command_context, **params):
 546     return run_web_platform_tests(command_context, **params)
 547
 548
 549 @Command(
 550     "web-platform-tests-update",
 551     category="testing",
 552     description="Update web-platform-test metadata.",
 553     parser=create_parser_update,
 554     virtualenv_name="wpt",
 555 )
 556 def update_web_platform_tests(command_context, **params):
 557     wpt_updater = command_context._spawn(WebPlatformTestsUpdater)
 558     logger = wpt_updater.setup_logging(**params)
 559     return wpt_updater.run_update(logger, **params)
 560
 561
 562 @Command(
 563     "wpt-update",
 564     category="testing",
 565     description="Update web-platform-test metadata.",
 566     parser=create_parser_update,
 567     virtualenv_name="wpt",
 568 )
 569 def update_wpt(command_context, **params):
 570     return update_web_platform_tests(command_context, **params)
 571
 572
 573 @Command(
 574     "wpt-manifest-update",
 575     category="testing",
 576     description="Update web-platform-test manifests.",
 577     parser=create_parser_manifest_update,
 578     virtualenv_name="wpt",
 579 )
 580 def wpt_manifest_update(command_context, **params):
 581     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 582     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 583     logger = wpt_runner.setup_logging(**params)
 584     logger.warning(
 585         "The wpt manifest is now automatically updated, "
 586         "so running this command is usually unnecessary"
 587     )
 588     return 0 if wpt_runner.update_manifest(logger, **params) else 1
 589
 590
 591 @Command(
 592     "wpt-serve",
 593     category="testing",
 594     description="Run the wpt server",
 595     parser=create_parser_serve,
 596     virtualenv_name="wpt",
 597 )
 598 def wpt_serve(command_context, **params):
 599     import logging
 600
 601     logger = logging.getLogger("web-platform-tests")
 602     logger.addHandler(logging.StreamHandler(sys.stdout))
 603     wpt_serve = command_context._spawn(WebPlatformTestsServeRunner)
 604     return wpt_serve.run(**params)
 605
 606
 607 @Command(
 608     "wpt-metadata-summary",
 609     category="testing",
 610     description="Create a json summary of the wpt metadata",
 611     parser=create_parser_metadata_summary,
 612     virtualenv_name="wpt",
 613 )
 614 def wpt_summary(command_context, **params):
 615     import metasummary
 616
 617     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 618     return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
 619
 620
 621 @Command(
 622     "wpt-metadata-merge",
 623     category="testing",
 624     parser=create_parser_metadata_merge,
 625     virtualenv_name="wpt",
 626 )
 627 def wpt_meta_merge(command_context, **params):
 628     import metamerge
 629
 630     if params["dest"] is None:
 631         params["dest"] = params["current"]
 632     return metamerge.run(**params)
 633
 634
 635 @Command(
 636     "wpt-unittest",
 637     category="testing",
 638     description="Run the wpt tools and wptrunner unit tests",
 639     parser=create_parser_unittest,
 640     virtualenv_name="wpt",
 641 )
 642 def wpt_unittest(command_context, **params):
 643     runner = command_context._spawn(WebPlatformTestsUnittestRunner)
 644     return 0 if runner.run(**params) else 1
 645
 646
 647 @Command(
 648     "wpt-test-paths",
 649     category="testing",
 650     description="Get a mapping from test ids to files",
 651     parser=create_parser_testpaths,
 652     virtualenv_name="wpt",
 653 )
 654 def wpt_test_paths(command_context, **params):
 655     runner = command_context._spawn(WebPlatformTestsTestPathsRunner)
 656     runner.run(**params)
 657     return 0
 658
 659
 660 @Command(
 661     "wpt-fission-regressions",
 662     category="testing",
 663     description="Dump a list of fission-specific regressions",
 664     parser=create_parser_fission_regressions,
 665     virtualenv_name="wpt",
 666 )
 667 def wpt_fission_regressions(command_context, **params):
 668     runner = command_context._spawn(WebPlatformTestsFissionRegressionsRunner)
 669     runner.run(**params)
 670     return 0
 671
 672
 673 @Command(
 674     "wpt-fetch-logs",
 675     category="testing",
 676     description="Fetch wptreport.json logs from taskcluster",
 677     parser=create_parser_fetch_logs,
 678     virtualenv_name="wpt-interop",
 679 )
 680 def wpt_fetch_logs(command_context, **params):
 681     import interop
 682
 683     interop.fetch_logs(**params)
 684     return 0
 685
 686
 687 @Command(
 688     "wpt-interop-score",
 689     category="testing",
 690     description="Score a run according to Interop 2023",
 691     parser=create_parser_interop_score,
 692     virtualenv_name="wpt-interop",
 693 )
 694 def wpt_interop_score(command_context, **params):
 695     import interop
 696
 697     interop.score_runs(**params)
 698     return 0