testing/web-platform/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the web-platform-tests test runner with mach.
   6
   7 import os
   8 import sys
   9
  10 from mach.decorators import Command
  11 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
  12 from mozbuild.base import MachCommandConditions as conditions
  13 from mozbuild.base import MozbuildObject
  14
  15 here = os.path.abspath(os.path.dirname(__file__))
  16 INTEROP_REQUIREMENTS_PATH = os.path.join(here, "interop_requirements.txt")
  17
  18
  19 class WebPlatformTestsRunnerSetup(MozbuildObject):
  20     default_log_type = "mach"
  21
  22     def __init__(self, *args, **kwargs):
  23         super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
  24         self._here = os.path.join(self.topsrcdir, "testing", "web-platform")
  25         kwargs["tests_root"] = os.path.join(self._here, "tests")
  26         sys.path.insert(0, kwargs["tests_root"])
  27         build_path = os.path.join(self.topobjdir, "build")
  28         if build_path not in sys.path:
  29             sys.path.append(build_path)
  30
  31     def kwargs_common(self, kwargs):
  32         """Setup kwargs relevant for all browser products"""
  33
  34         tests_src_path = os.path.join(self._here, "tests")
  35
  36         if (
  37             kwargs["product"] in {"firefox", "firefox_android"}
  38             and kwargs["specialpowers_path"] is None
  39         ):
  40             kwargs["specialpowers_path"] = os.path.join(
  41                 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi"
  42             )
  43
  44         if kwargs["product"] == "firefox_android":
  45             # package_name may be different in the future
  46             package_name = kwargs["package_name"]
  47             if not package_name:
  48                 kwargs[
  49                     "package_name"
  50                 ] = package_name = "org.mozilla.geckoview.test_runner"
  51
  52             # Note that this import may fail in non-firefox-for-android trees
  53             from mozrunner.devices.android_device import (
  54                 InstallIntent,
  55                 get_adb_path,
  56                 verify_android_device,
  57             )
  58
  59             kwargs["adb_binary"] = get_adb_path(self)
  60             install = (
  61                 InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES
  62             )
  63             verify_android_device(
  64                 self, install=install, verbose=False, xre=True, app=package_name
  65             )
  66
  67             if kwargs["certutil_binary"] is None:
  68                 kwargs["certutil_binary"] = os.path.join(
  69                     os.environ.get("MOZ_HOST_BIN"), "certutil"
  70                 )
  71
  72             if kwargs["install_fonts"] is None:
  73                 kwargs["install_fonts"] = True
  74
  75             if not kwargs["device_serial"]:
  76                 kwargs["device_serial"] = ["emulator-5554"]
  77
  78         if kwargs["config"] is None:
  79             kwargs["config"] = os.path.join(
  80                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
  81             )
  82
  83         if (
  84             kwargs["exclude"] is None
  85             and kwargs["include"] is None
  86             and not sys.platform.startswith("linux")
  87         ):
  88             kwargs["exclude"] = ["css"]
  89
  90         if kwargs["ssl_type"] in (None, "pregenerated"):
  91             cert_root = os.path.join(tests_src_path, "tools", "certs")
  92             if kwargs["ca_cert_path"] is None:
  93                 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
  94
  95             if kwargs["host_key_path"] is None:
  96                 kwargs["host_key_path"] = os.path.join(
  97                     cert_root, "web-platform.test.key"
  98                 )
  99
 100             if kwargs["host_cert_path"] is None:
 101                 kwargs["host_cert_path"] = os.path.join(
 102                     cert_root, "web-platform.test.pem"
 103                 )
 104
 105         if kwargs["reftest_screenshot"] is None:
 106             kwargs["reftest_screenshot"] = "fail"
 107
 108         kwargs["capture_stdio"] = True
 109
 110         return kwargs
 111
 112     def kwargs_firefox(self, kwargs):
 113         """Setup kwargs specific to running Firefox and other gecko browsers"""
 114         import mozinfo
 115         from wptrunner import wptcommandline
 116
 117         kwargs = self.kwargs_common(kwargs)
 118
 119         if kwargs["binary"] is None:
 120             kwargs["binary"] = self.get_binary_path()
 121
 122         if kwargs["certutil_binary"] is None:
 123             kwargs["certutil_binary"] = self.get_binary_path("certutil")
 124
 125         if kwargs["webdriver_binary"] is None:
 126             try_paths = [self.get_binary_path("geckodriver", validate_exists=False)]
 127             ext = ".exe" if sys.platform in ["win32", "msys", "cygwin"] else ""
 128             for build_type in ["release", "debug"]:
 129                 try_paths.append(
 130                     os.path.join(
 131                         self.topsrcdir, "target", build_type, f"geckodriver{ext}"
 132                     )
 133                 )
 134             found_paths = []
 135             for path in try_paths:
 136                 if os.path.exists(path):
 137                     found_paths.append(path)
 138
 139             if found_paths:
 140                 # Pick the most recently modified version
 141                 found_paths.sort(key=os.path.getmtime)
 142                 kwargs["webdriver_binary"] = found_paths[-1]
 143
 144         if kwargs["install_fonts"] is None:
 145             kwargs["install_fonts"] = True
 146
 147         if (
 148             kwargs["install_fonts"]
 149             and mozinfo.info["os"] == "win"
 150             and mozinfo.info["os_version"] == "6.1"
 151         ):
 152             # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
 153             self.setup_fonts_firefox()
 154             kwargs["install_fonts"] = False
 155
 156         if kwargs["preload_browser"] is None:
 157             kwargs["preload_browser"] = False
 158
 159         if kwargs["prefs_root"] is None:
 160             kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles")
 161
 162         if kwargs["stackfix_dir"] is None:
 163             kwargs["stackfix_dir"] = self.bindir
 164
 165         kwargs = wptcommandline.check_args(kwargs)
 166
 167         return kwargs
 168
 169     def kwargs_wptrun(self, kwargs):
 170         """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
 171         from tools.wpt import run, virtualenv
 172
 173         kwargs = self.kwargs_common(kwargs)
 174
 175         # Our existing kwargs corresponds to the wptrunner command line arguments.
 176         # `wpt run` extends this with some additional arguments that are consumed by
 177         # the frontend. Copy over the default values of these extra arguments so they
 178         # are present when we call into that frontend.
 179         run_parser = run.create_parser()
 180         run_kwargs = run_parser.parse_args([kwargs["product"], kwargs["test_list"]])
 181
 182         for key, value in vars(run_kwargs).items():
 183             if key not in kwargs:
 184                 kwargs[key] = value
 185
 186         # Install the deps
 187         # We do this explicitly to avoid calling pip with options that aren't
 188         # supported in the in-tree version
 189         wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
 190         browser_cls = run.product_setup[kwargs["product"]].browser_cls
 191         requirements = ["requirements.txt"]
 192         if hasattr(browser_cls, "requirements"):
 193             requirements.append(browser_cls.requirements)
 194
 195         for filename in requirements:
 196             path = os.path.join(wptrunner_path, filename)
 197             if os.path.exists(path):
 198                 self.virtualenv_manager.install_pip_requirements(
 199                     path, require_hashes=False
 200                 )
 201
 202         venv = virtualenv.Virtualenv(
 203             self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
 204         )
 205         try:
 206             browser_cls, kwargs = run.setup_wptrunner(venv, **kwargs)
 207         except run.WptrunError as e:
 208             print(e, file=sys.stderr)
 209             sys.exit(1)
 210
 211         # This is kind of a hack; override the metadata paths so we don't use
 212         # gecko metadata for non-gecko products
 213         for url_base, test_root in kwargs["test_paths"].items():
 214             meta_suffix = url_base.strip("/")
 215             meta_dir = os.path.join(
 216                 self._here, "products", kwargs["product"].name, meta_suffix
 217             )
 218             test_root.metadata_path = meta_dir
 219             if not os.path.exists(meta_dir):
 220                 os.makedirs(meta_dir)
 221         return kwargs
 222
 223     def setup_fonts_firefox(self):
 224         # Ensure the Ahem font is available
 225         if not sys.platform.startswith("darwin"):
 226             font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
 227         else:
 228             font_path = os.path.join(
 229                 os.path.dirname(self.get_binary_path()),
 230                 os.pardir,
 231                 "Resources",
 232                 "res",
 233                 "fonts",
 234             )
 235         ahem_src = os.path.join(
 236             self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
 237         )
 238         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 239         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
 240             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
 241                 dest.write(src.read())
 242
 243
 244 class WebPlatformTestsServeRunner(MozbuildObject):
 245     def run(self, **kwargs):
 246         sys.path.insert(0, os.path.join(here, "tests"))
 247         sys.path.insert(0, os.path.join(here, "tests", "tools"))
 248         import logging
 249
 250         import manifestupdate
 251         from serve import serve
 252         from wptrunner import wptcommandline
 253
 254         logger = logging.getLogger("web-platform-tests")
 255
 256         src_root = self.topsrcdir
 257         obj_root = self.topobjdir
 258         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 259
 260         config_path = manifestupdate.generate_config(
 261             logger,
 262             src_root,
 263             src_wpt_dir,
 264             os.path.join(obj_root, "_tests", "web-platform"),
 265             False,
 266         )
 267
 268         test_paths = wptcommandline.get_test_paths(
 269             wptcommandline.config.read(config_path)
 270         )
 271
 272         def get_route_builder(*args, **kwargs):
 273             route_builder = serve.get_route_builder(*args, **kwargs)
 274
 275             for url_base, paths in test_paths.items():
 276                 if url_base != "/":
 277                     route_builder.add_mount_point(url_base, paths.tests_path)
 278
 279             return route_builder
 280
 281         return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1
 282
 283
 284 class WebPlatformTestsUpdater(MozbuildObject):
 285     """Update web platform tests."""
 286
 287     def setup_logging(self, **kwargs):
 288         import update
 289
 290         return update.setup_logging(kwargs, {"mach": sys.stdout})
 291
 292     def update_manifest(self, logger, **kwargs):
 293         import manifestupdate
 294
 295         return manifestupdate.run(
 296             logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs
 297         )
 298
 299     def run_update(self, logger, **kwargs):
 300         import update
 301         from update import updatecommandline
 302
 303         self.update_manifest(logger, **kwargs)
 304
 305         if kwargs["config"] is None:
 306             kwargs["config"] = os.path.join(
 307                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
 308             )
 309         if kwargs["product"] is None:
 310             kwargs["product"] = "firefox"
 311
 312         kwargs["store_state"] = False
 313
 314         kwargs = updatecommandline.check_args(kwargs)
 315
 316         try:
 317             update.run_update(logger, **kwargs)
 318         except Exception:
 319             import traceback
 320
 321             traceback.print_exc()
 322
 323
 324 class WebPlatformTestsUnittestRunner(MozbuildObject):
 325     def run(self, **kwargs):
 326         import unittestrunner
 327
 328         return unittestrunner.run(self.topsrcdir, **kwargs)
 329
 330
 331 class WebPlatformTestsTestPathsRunner(MozbuildObject):
 332     """Update web platform tests."""
 333
 334     def run(self, **kwargs):
 335         sys.path.insert(
 336             0,
 337             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 338         )
 339         import logging
 340
 341         import manifestupdate
 342         from manifest import testpaths
 343         from wptrunner import wptcommandline
 344
 345         logger = logging.getLogger("web-platform-tests")
 346
 347         src_root = self.topsrcdir
 348         obj_root = self.topobjdir
 349         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 350
 351         config_path = manifestupdate.generate_config(
 352             logger,
 353             src_root,
 354             src_wpt_dir,
 355             os.path.join(obj_root, "_tests", "web-platform"),
 356             False,
 357         )
 358
 359         test_paths = wptcommandline.get_test_paths(
 360             wptcommandline.config.read(config_path)
 361         )
 362         results = {}
 363         for url_base, paths in test_paths.items():
 364             if "manifest_path" not in paths:
 365                 paths["manifest_path"] = os.path.join(
 366                     paths["metadata_path"], "MANIFEST.json"
 367                 )
 368             results.update(
 369                 testpaths.get_paths(
 370                     path=paths["manifest_path"],
 371                     src_root=src_root,
 372                     tests_root=paths["tests_path"],
 373                     update=kwargs["update"],
 374                     rebuild=kwargs["rebuild"],
 375                     url_base=url_base,
 376                     cache_root=kwargs["cache_root"],
 377                     test_ids=kwargs["test_ids"],
 378                 )
 379             )
 380         testpaths.write_output(results, kwargs["json"])
 381         return True
 382
 383
 384 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject):
 385     def run(self, **kwargs):
 386         import fissionregressions
 387         import mozlog
 388
 389         src_root = self.topsrcdir
 390         obj_root = self.topobjdir
 391         logger = mozlog.structuredlog.StructuredLogger("web-platform-tests")
 392
 393         try:
 394             return fissionregressions.run(logger, src_root, obj_root, **kwargs)
 395         except Exception:
 396             import pdb
 397             import traceback
 398
 399             traceback.print_exc()
 400             pdb.post_mortem()
 401
 402
 403 def create_parser_update():
 404     from update import updatecommandline
 405
 406     return updatecommandline.create_parser()
 407
 408
 409 def create_parser_manifest_update():
 410     import manifestupdate
 411
 412     return manifestupdate.create_parser()
 413
 414
 415 def create_parser_metadata_summary():
 416     import metasummary
 417
 418     return metasummary.create_parser()
 419
 420
 421 def create_parser_metadata_merge():
 422     import metamerge
 423
 424     return metamerge.get_parser()
 425
 426
 427 def create_parser_serve():
 428     sys.path.insert(
 429         0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools"))
 430     )
 431     import serve
 432
 433     return serve.serve.get_parser()
 434
 435
 436 def create_parser_unittest():
 437     import unittestrunner
 438
 439     return unittestrunner.get_parser()
 440
 441
 442 def create_parser_fission_regressions():
 443     import fissionregressions
 444
 445     return fissionregressions.get_parser()
 446
 447
 448 def create_parser_fetch_logs():
 449     import interop
 450
 451     return interop.get_parser_fetch_logs()
 452
 453
 454 def create_parser_interop_score():
 455     import interop
 456
 457     return interop.get_parser_interop_score()
 458
 459
 460 def create_parser_testpaths():
 461     import argparse
 462
 463     from mach.util import get_state_dir
 464
 465     parser = argparse.ArgumentParser()
 466     parser.add_argument(
 467         "--no-update",
 468         dest="update",
 469         action="store_false",
 470         default=True,
 471         help="Don't update manifest before continuing",
 472     )
 473     parser.add_argument(
 474         "-r",
 475         "--rebuild",
 476         action="store_true",
 477         default=False,
 478         help="Force a full rebuild of the manifest.",
 479     )
 480     parser.add_argument(
 481         "--cache-root",
 482         action="store",
 483         default=os.path.join(get_state_dir(), "cache", "wpt"),
 484         help="Path in which to store any caches (default <tests_root>/.wptcache/)",
 485     )
 486     parser.add_argument(
 487         "test_ids", action="store", nargs="+", help="Test ids for which to get paths"
 488     )
 489     parser.add_argument(
 490         "--json", action="store_true", default=False, help="Output as JSON"
 491     )
 492     return parser
 493
 494
 495 @Command(
 496     "web-platform-tests",
 497     category="testing",
 498     conditions=[conditions.is_firefox_or_android],
 499     description="Run web-platform-tests.",
 500     parser=create_parser_wpt,
 501     virtualenv_name="wpt",
 502 )
 503 def run_web_platform_tests(command_context, **params):
 504     if params["product"] is None:
 505         if conditions.is_android(command_context):
 506             params["product"] = "firefox_android"
 507         else:
 508             params["product"] = "firefox"
 509     if "test_objects" in params:
 510         include = []
 511         test_types = set()
 512         for item in params["test_objects"]:
 513             include.append(item["name"])
 514             test_types.add(item.get("subsuite"))
 515         if None not in test_types:
 516             params["test_types"] = list(test_types)
 517         params["include"] = include
 518         del params["test_objects"]
 519         # subsuite coming from `mach test` means something more like `test type`, so remove that argument
 520         if "subsuite" in params:
 521             del params["subsuite"]
 522     if params.get("debugger", None):
 523         import mozdebug
 524
 525         if not mozdebug.get_debugger_info(params.get("debugger")):
 526             sys.exit(1)
 527
 528     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 529     wpt_setup._mach_context = command_context._mach_context
 530     wpt_setup._virtualenv_name = command_context._virtualenv_name
 531     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 532
 533     logger = wpt_runner.setup_logging(**params)
 534     # wptrunner already handles setting any log parameter from
 535     # mach test to the logger, so it's OK to remove that argument now
 536     if "log" in params:
 537         del params["log"]
 538
 539     if (
 540         conditions.is_android(command_context)
 541         and params["product"] != "firefox_android"
 542     ):
 543         logger.warning("Must specify --product=firefox_android in Android environment.")
 544
 545     return wpt_runner.run(logger, **params)
 546
 547
 548 @Command(
 549     "wpt",
 550     category="testing",
 551     conditions=[conditions.is_firefox_or_android],
 552     description="Run web-platform-tests.",
 553     parser=create_parser_wpt,
 554     virtualenv_name="wpt",
 555 )
 556 def run_wpt(command_context, **params):
 557     return run_web_platform_tests(command_context, **params)
 558
 559
 560 @Command(
 561     "web-platform-tests-update",
 562     category="testing",
 563     description="Update web-platform-test metadata.",
 564     parser=create_parser_update,
 565     virtualenv_name="wpt",
 566 )
 567 def update_web_platform_tests(command_context, **params):
 568     wpt_updater = command_context._spawn(WebPlatformTestsUpdater)
 569     logger = wpt_updater.setup_logging(**params)
 570     return wpt_updater.run_update(logger, **params)
 571
 572
 573 @Command(
 574     "wpt-update",
 575     category="testing",
 576     description="Update web-platform-test metadata.",
 577     parser=create_parser_update,
 578     virtualenv_name="wpt",
 579 )
 580 def update_wpt(command_context, **params):
 581     return update_web_platform_tests(command_context, **params)
 582
 583
 584 @Command(
 585     "wpt-manifest-update",
 586     category="testing",
 587     description="Update web-platform-test manifests.",
 588     parser=create_parser_manifest_update,
 589     virtualenv_name="wpt",
 590 )
 591 def wpt_manifest_update(command_context, **params):
 592     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 593     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 594     logger = wpt_runner.setup_logging(**params)
 595     logger.warning(
 596         "The wpt manifest is now automatically updated, "
 597         "so running this command is usually unnecessary"
 598     )
 599     return 0 if wpt_runner.update_manifest(logger, **params) else 1
 600
 601
 602 @Command(
 603     "wpt-serve",
 604     category="testing",
 605     description="Run the wpt server",
 606     parser=create_parser_serve,
 607     virtualenv_name="wpt",
 608 )
 609 def wpt_serve(command_context, **params):
 610     import logging
 611
 612     logger = logging.getLogger("web-platform-tests")
 613     logger.addHandler(logging.StreamHandler(sys.stdout))
 614     wpt_serve = command_context._spawn(WebPlatformTestsServeRunner)
 615     return wpt_serve.run(**params)
 616
 617
 618 @Command(
 619     "wpt-metadata-summary",
 620     category="testing",
 621     description="Create a json summary of the wpt metadata",
 622     parser=create_parser_metadata_summary,
 623     virtualenv_name="wpt",
 624 )
 625 def wpt_summary(command_context, **params):
 626     import metasummary
 627
 628     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 629     return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
 630
 631
 632 @Command(
 633     "wpt-metadata-merge",
 634     category="testing",
 635     parser=create_parser_metadata_merge,
 636     virtualenv_name="wpt",
 637 )
 638 def wpt_meta_merge(command_context, **params):
 639     import metamerge
 640
 641     if params["dest"] is None:
 642         params["dest"] = params["current"]
 643     return metamerge.run(**params)
 644
 645
 646 @Command(
 647     "wpt-unittest",
 648     category="testing",
 649     description="Run the wpt tools and wptrunner unit tests",
 650     parser=create_parser_unittest,
 651     virtualenv_name="wpt",
 652 )
 653 def wpt_unittest(command_context, **params):
 654     runner = command_context._spawn(WebPlatformTestsUnittestRunner)
 655     return 0 if runner.run(**params) else 1
 656
 657
 658 @Command(
 659     "wpt-test-paths",
 660     category="testing",
 661     description="Get a mapping from test ids to files",
 662     parser=create_parser_testpaths,
 663     virtualenv_name="wpt",
 664 )
 665 def wpt_test_paths(command_context, **params):
 666     runner = command_context._spawn(WebPlatformTestsTestPathsRunner)
 667     runner.run(**params)
 668     return 0
 669
 670
 671 @Command(
 672     "wpt-fission-regressions",
 673     category="testing",
 674     description="Dump a list of fission-specific regressions",
 675     parser=create_parser_fission_regressions,
 676     virtualenv_name="wpt",
 677 )
 678 def wpt_fission_regressions(command_context, **params):
 679     runner = command_context._spawn(WebPlatformTestsFissionRegressionsRunner)
 680     runner.run(**params)
 681     return 0
 682
 683
 684 @Command(
 685     "wpt-fetch-logs",
 686     category="testing",
 687     description="Fetch wptreport.json logs from taskcluster",
 688     parser=create_parser_fetch_logs,
 689     virtualenv_name="wpt-interop",
 690 )
 691 def wpt_fetch_logs(command_context, **params):
 692     import interop
 693
 694     interop.fetch_logs(**params)
 695     return 0
 696
 697
 698 @Command(
 699     "wpt-interop-score",
 700     category="testing",
 701     description="Score a run according to Interop 2023",
 702     parser=create_parser_interop_score,
 703     virtualenv_name="wpt-interop",
 704 )
 705 def wpt_interop_score(command_context, **params):
 706     import interop
 707
 708     interop.score_runs(**params)
 709     return 0