testing/web-platform/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the web-platform-tests test runner with mach.
   6
   7 import os
   8 import sys
   9
  10 from mach.decorators import Command
  11 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
  12 from mozbuild.base import MachCommandConditions as conditions
  13 from mozbuild.base import MozbuildObject
  14
  15 here = os.path.abspath(os.path.dirname(__file__))
  16 INTEROP_REQUIREMENTS_PATH = os.path.join(here, "interop_requirements.txt")
  17
  18
  19 class WebPlatformTestsRunnerSetup(MozbuildObject):
  20     default_log_type = "mach"
  21
  22     def __init__(self, *args, **kwargs):
  23         super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
  24         self._here = os.path.join(self.topsrcdir, "testing", "web-platform")
  25         kwargs["tests_root"] = os.path.join(self._here, "tests")
  26         sys.path.insert(0, kwargs["tests_root"])
  27         build_path = os.path.join(self.topobjdir, "build")
  28         if build_path not in sys.path:
  29             sys.path.append(build_path)
  30
  31     def kwargs_common(self, kwargs):
  32         """Setup kwargs relevant for all browser products"""
  33
  34         tests_src_path = os.path.join(self._here, "tests")
  35
  36         if (
  37             kwargs["product"] in {"firefox", "firefox_android"}
  38             and kwargs["specialpowers_path"] is None
  39         ):
  40             kwargs["specialpowers_path"] = os.path.join(
  41                 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi"
  42             )
  43
  44         if kwargs["product"] == "firefox_android":
  45             # package_name may be different in the future
  46             package_name = kwargs["package_name"]
  47             if not package_name:
  48                 kwargs[
  49                     "package_name"
  50                 ] = package_name = "org.mozilla.geckoview.test_runner"
  51
  52             # Note that this import may fail in non-firefox-for-android trees
  53             from mozrunner.devices.android_device import (
  54                 InstallIntent,
  55                 get_adb_path,
  56                 verify_android_device,
  57             )
  58
  59             kwargs["adb_binary"] = get_adb_path(self)
  60             install = (
  61                 InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES
  62             )
  63             verify_android_device(
  64                 self, install=install, verbose=False, xre=True, app=package_name
  65             )
  66
  67             if kwargs["certutil_binary"] is None:
  68                 kwargs["certutil_binary"] = os.path.join(
  69                     os.environ.get("MOZ_HOST_BIN"), "certutil"
  70                 )
  71
  72             if kwargs["install_fonts"] is None:
  73                 kwargs["install_fonts"] = True
  74
  75             if not kwargs["device_serial"]:
  76                 kwargs["device_serial"] = ["emulator-5554"]
  77
  78         if kwargs["config"] is None:
  79             kwargs["config"] = os.path.join(
  80                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
  81             )
  82
  83         if (
  84             kwargs["exclude"] is None
  85             and kwargs["include"] is None
  86             and not sys.platform.startswith("linux")
  87         ):
  88             kwargs["exclude"] = ["css"]
  89
  90         if kwargs["ssl_type"] in (None, "pregenerated"):
  91             cert_root = os.path.join(tests_src_path, "tools", "certs")
  92             if kwargs["ca_cert_path"] is None:
  93                 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
  94
  95             if kwargs["host_key_path"] is None:
  96                 kwargs["host_key_path"] = os.path.join(
  97                     cert_root, "web-platform.test.key"
  98                 )
  99
 100             if kwargs["host_cert_path"] is None:
 101                 kwargs["host_cert_path"] = os.path.join(
 102                     cert_root, "web-platform.test.pem"
 103                 )
 104
 105         if kwargs["reftest_screenshot"] is None:
 106             kwargs["reftest_screenshot"] = "fail"
 107
 108         kwargs["capture_stdio"] = True
 109
 110         return kwargs
 111
 112     def kwargs_firefox(self, kwargs):
 113         """Setup kwargs specific to running Firefox and other gecko browsers"""
 114         from wptrunner import wptcommandline
 115
 116         kwargs = self.kwargs_common(kwargs)
 117
 118         if kwargs["binary"] is None:
 119             kwargs["binary"] = self.get_binary_path()
 120
 121         if kwargs["certutil_binary"] is None:
 122             kwargs["certutil_binary"] = self.get_binary_path("certutil")
 123
 124         if kwargs["webdriver_binary"] is None:
 125             try_paths = [self.get_binary_path("geckodriver", validate_exists=False)]
 126             ext = ".exe" if sys.platform in ["win32", "msys", "cygwin"] else ""
 127             for build_type in ["release", "debug"]:
 128                 try_paths.append(
 129                     os.path.join(
 130                         self.topsrcdir, "target", build_type, f"geckodriver{ext}"
 131                     )
 132                 )
 133             found_paths = []
 134             for path in try_paths:
 135                 if os.path.exists(path):
 136                     found_paths.append(path)
 137
 138             if found_paths:
 139                 # Pick the most recently modified version
 140                 found_paths.sort(key=os.path.getmtime)
 141                 kwargs["webdriver_binary"] = found_paths[-1]
 142
 143         if kwargs["install_fonts"] is None:
 144             kwargs["install_fonts"] = True
 145
 146         if kwargs["preload_browser"] is None:
 147             kwargs["preload_browser"] = False
 148
 149         if kwargs["prefs_root"] is None:
 150             kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles")
 151
 152         if kwargs["stackfix_dir"] is None:
 153             kwargs["stackfix_dir"] = self.bindir
 154
 155         kwargs = wptcommandline.check_args(kwargs)
 156
 157         return kwargs
 158
 159     def kwargs_wptrun(self, kwargs):
 160         """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
 161         from tools.wpt import run, virtualenv
 162
 163         kwargs = self.kwargs_common(kwargs)
 164
 165         # Our existing kwargs corresponds to the wptrunner command line arguments.
 166         # `wpt run` extends this with some additional arguments that are consumed by
 167         # the frontend. Copy over the default values of these extra arguments so they
 168         # are present when we call into that frontend.
 169         run_parser = run.create_parser()
 170         run_kwargs = run_parser.parse_args([kwargs["product"], kwargs["test_list"]])
 171
 172         for key, value in vars(run_kwargs).items():
 173             if key not in kwargs:
 174                 kwargs[key] = value
 175
 176         # Install the deps
 177         # We do this explicitly to avoid calling pip with options that aren't
 178         # supported in the in-tree version
 179         wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
 180         browser_cls = run.product_setup[kwargs["product"]].browser_cls
 181         requirements = ["requirements.txt"]
 182         if (
 183             hasattr(browser_cls, "requirements")
 184             and browser_cls.requirements is not None
 185         ):
 186             requirements.append(browser_cls.requirements)
 187
 188         for filename in requirements:
 189             path = os.path.join(wptrunner_path, filename)
 190             if os.path.exists(path):
 191                 self.virtualenv_manager.install_pip_requirements(
 192                     path, require_hashes=False
 193                 )
 194
 195         venv = virtualenv.Virtualenv(
 196             self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
 197         )
 198         try:
 199             browser_cls, kwargs = run.setup_wptrunner(venv, **kwargs)
 200         except run.WptrunError as e:
 201             print(e, file=sys.stderr)
 202             sys.exit(1)
 203
 204         # This is kind of a hack; override the metadata paths so we don't use
 205         # gecko metadata for non-gecko products
 206         for url_base, test_root in kwargs["test_paths"].items():
 207             meta_suffix = url_base.strip("/")
 208             meta_dir = os.path.join(
 209                 self._here, "products", kwargs["product"].name, meta_suffix
 210             )
 211             test_root.metadata_path = meta_dir
 212             if not os.path.exists(meta_dir):
 213                 os.makedirs(meta_dir)
 214         return kwargs
 215
 216     def setup_fonts_firefox(self):
 217         # Ensure the Ahem font is available
 218         if not sys.platform.startswith("darwin"):
 219             font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
 220         else:
 221             font_path = os.path.join(
 222                 os.path.dirname(self.get_binary_path()),
 223                 os.pardir,
 224                 "Resources",
 225                 "res",
 226                 "fonts",
 227             )
 228         ahem_src = os.path.join(
 229             self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
 230         )
 231         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 232         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
 233             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
 234                 dest.write(src.read())
 235
 236
 237 class WebPlatformTestsServeRunner(MozbuildObject):
 238     def run(self, **kwargs):
 239         sys.path.insert(0, os.path.join(here, "tests"))
 240         sys.path.insert(0, os.path.join(here, "tests", "tools"))
 241         import logging
 242
 243         import manifestupdate
 244         from serve import serve
 245         from wptrunner import wptcommandline
 246
 247         logger = logging.getLogger("web-platform-tests")
 248
 249         src_root = self.topsrcdir
 250         obj_root = self.topobjdir
 251         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 252
 253         config_path = manifestupdate.generate_config(
 254             logger,
 255             src_root,
 256             src_wpt_dir,
 257             os.path.join(obj_root, "_tests", "web-platform"),
 258             False,
 259         )
 260
 261         test_paths = wptcommandline.get_test_paths(
 262             wptcommandline.config.read(config_path)
 263         )
 264
 265         def get_route_builder(*args, **kwargs):
 266             route_builder = serve.get_route_builder(*args, **kwargs)
 267
 268             for url_base, paths in test_paths.items():
 269                 if url_base != "/":
 270                     route_builder.add_mount_point(url_base, paths.tests_path)
 271
 272             return route_builder
 273
 274         return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1
 275
 276
 277 class WebPlatformTestsUpdater(MozbuildObject):
 278     """Update web platform tests."""
 279
 280     def setup_logging(self, **kwargs):
 281         import update
 282
 283         return update.setup_logging(kwargs, {"mach": sys.stdout})
 284
 285     def update_manifest(self, logger, **kwargs):
 286         import manifestupdate
 287
 288         return manifestupdate.run(
 289             logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs
 290         )
 291
 292     def run_update(self, logger, **kwargs):
 293         import update
 294         from update import updatecommandline
 295
 296         self.update_manifest(logger, **kwargs)
 297
 298         if kwargs["config"] is None:
 299             kwargs["config"] = os.path.join(
 300                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
 301             )
 302         if kwargs["product"] is None:
 303             kwargs["product"] = "firefox"
 304
 305         kwargs["store_state"] = False
 306
 307         kwargs = updatecommandline.check_args(kwargs)
 308
 309         try:
 310             update.run_update(logger, **kwargs)
 311         except Exception:
 312             import traceback
 313
 314             traceback.print_exc()
 315
 316
 317 class WebPlatformTestsUnittestRunner(MozbuildObject):
 318     def run(self, **kwargs):
 319         import unittestrunner
 320
 321         return unittestrunner.run(self.topsrcdir, **kwargs)
 322
 323
 324 class WebPlatformTestsTestPathsRunner(MozbuildObject):
 325     """Update web platform tests."""
 326
 327     def run(self, **kwargs):
 328         sys.path.insert(
 329             0,
 330             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 331         )
 332         import logging
 333
 334         import manifestupdate
 335         from manifest import testpaths
 336         from wptrunner import wptcommandline
 337
 338         logger = logging.getLogger("web-platform-tests")
 339
 340         src_root = self.topsrcdir
 341         obj_root = self.topobjdir
 342         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 343
 344         config_path = manifestupdate.generate_config(
 345             logger,
 346             src_root,
 347             src_wpt_dir,
 348             os.path.join(obj_root, "_tests", "web-platform"),
 349             False,
 350         )
 351
 352         test_paths = wptcommandline.get_test_paths(
 353             wptcommandline.config.read(config_path)
 354         )
 355         results = {}
 356         for url_base, paths in test_paths.items():
 357             if "manifest_path" not in paths:
 358                 paths["manifest_path"] = os.path.join(
 359                     paths["metadata_path"], "MANIFEST.json"
 360                 )
 361             results.update(
 362                 testpaths.get_paths(
 363                     path=paths["manifest_path"],
 364                     src_root=src_root,
 365                     tests_root=paths["tests_path"],
 366                     update=kwargs["update"],
 367                     rebuild=kwargs["rebuild"],
 368                     url_base=url_base,
 369                     cache_root=kwargs["cache_root"],
 370                     test_ids=kwargs["test_ids"],
 371                 )
 372             )
 373         testpaths.write_output(results, kwargs["json"])
 374         return True
 375
 376
 377 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject):
 378     def run(self, **kwargs):
 379         import fissionregressions
 380         import mozlog
 381
 382         src_root = self.topsrcdir
 383         obj_root = self.topobjdir
 384         logger = mozlog.structuredlog.StructuredLogger("web-platform-tests")
 385
 386         try:
 387             return fissionregressions.run(logger, src_root, obj_root, **kwargs)
 388         except Exception:
 389             import pdb
 390             import traceback
 391
 392             traceback.print_exc()
 393             pdb.post_mortem()
 394
 395
 396 def create_parser_update():
 397     from update import updatecommandline
 398
 399     return updatecommandline.create_parser()
 400
 401
 402 def create_parser_manifest_update():
 403     import manifestupdate
 404
 405     return manifestupdate.create_parser()
 406
 407
 408 def create_parser_metadata_summary():
 409     import metasummary
 410
 411     return metasummary.create_parser()
 412
 413
 414 def create_parser_metadata_merge():
 415     import metamerge
 416
 417     return metamerge.get_parser()
 418
 419
 420 def create_parser_serve():
 421     sys.path.insert(
 422         0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools"))
 423     )
 424     import serve
 425
 426     return serve.serve.get_parser()
 427
 428
 429 def create_parser_unittest():
 430     import unittestrunner
 431
 432     return unittestrunner.get_parser()
 433
 434
 435 def create_parser_fission_regressions():
 436     import fissionregressions
 437
 438     return fissionregressions.get_parser()
 439
 440
 441 def create_parser_fetch_logs():
 442     import interop
 443
 444     return interop.get_parser_fetch_logs()
 445
 446
 447 def create_parser_interop_score():
 448     import interop
 449
 450     return interop.get_parser_interop_score()
 451
 452
 453 def create_parser_testpaths():
 454     import argparse
 455
 456     from mach.util import get_state_dir
 457
 458     parser = argparse.ArgumentParser()
 459     parser.add_argument(
 460         "--no-update",
 461         dest="update",
 462         action="store_false",
 463         default=True,
 464         help="Don't update manifest before continuing",
 465     )
 466     parser.add_argument(
 467         "-r",
 468         "--rebuild",
 469         action="store_true",
 470         default=False,
 471         help="Force a full rebuild of the manifest.",
 472     )
 473     parser.add_argument(
 474         "--cache-root",
 475         action="store",
 476         default=os.path.join(get_state_dir(), "cache", "wpt"),
 477         help="Path in which to store any caches (default <tests_root>/.wptcache/)",
 478     )
 479     parser.add_argument(
 480         "test_ids", action="store", nargs="+", help="Test ids for which to get paths"
 481     )
 482     parser.add_argument(
 483         "--json", action="store_true", default=False, help="Output as JSON"
 484     )
 485     return parser
 486
 487
 488 @Command(
 489     "web-platform-tests",
 490     category="testing",
 491     conditions=[conditions.is_firefox_or_android],
 492     description="Run web-platform-tests.",
 493     parser=create_parser_wpt,
 494     virtualenv_name="wpt",
 495 )
 496 def run_web_platform_tests(command_context, **params):
 497     if params["product"] is None:
 498         if conditions.is_android(command_context):
 499             params["product"] = "firefox_android"
 500         else:
 501             params["product"] = "firefox"
 502     if "test_objects" in params:
 503         include = []
 504         test_types = set()
 505         for item in params["test_objects"]:
 506             include.append(item["name"])
 507             test_types.add(item.get("subsuite"))
 508         if None not in test_types:
 509             params["test_types"] = list(test_types)
 510         params["include"] = include
 511         del params["test_objects"]
 512         # subsuite coming from `mach test` means something more like `test type`, so remove that argument
 513         if "subsuite" in params:
 514             del params["subsuite"]
 515     if params.get("debugger", None):
 516         import mozdebug
 517
 518         if not mozdebug.get_debugger_info(params.get("debugger")):
 519             sys.exit(1)
 520
 521     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 522     wpt_setup._mach_context = command_context._mach_context
 523     wpt_setup._virtualenv_name = command_context._virtualenv_name
 524     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 525
 526     logger = wpt_runner.setup_logging(**params)
 527     # wptrunner already handles setting any log parameter from
 528     # mach test to the logger, so it's OK to remove that argument now
 529     if "log" in params:
 530         del params["log"]
 531
 532     if (
 533         conditions.is_android(command_context)
 534         and params["product"] != "firefox_android"
 535     ):
 536         logger.warning("Must specify --product=firefox_android in Android environment.")
 537
 538     return wpt_runner.run(logger, **params)
 539
 540
 541 @Command(
 542     "wpt",
 543     category="testing",
 544     conditions=[conditions.is_firefox_or_android],
 545     description="Run web-platform-tests.",
 546     parser=create_parser_wpt,
 547     virtualenv_name="wpt",
 548 )
 549 def run_wpt(command_context, **params):
 550     return run_web_platform_tests(command_context, **params)
 551
 552
 553 @Command(
 554     "web-platform-tests-update",
 555     category="testing",
 556     description="Update web-platform-test metadata.",
 557     parser=create_parser_update,
 558     virtualenv_name="wpt",
 559 )
 560 def update_web_platform_tests(command_context, **params):
 561     wpt_updater = command_context._spawn(WebPlatformTestsUpdater)
 562     logger = wpt_updater.setup_logging(**params)
 563     return wpt_updater.run_update(logger, **params)
 564
 565
 566 @Command(
 567     "wpt-update",
 568     category="testing",
 569     description="Update web-platform-test metadata.",
 570     parser=create_parser_update,
 571     virtualenv_name="wpt",
 572 )
 573 def update_wpt(command_context, **params):
 574     return update_web_platform_tests(command_context, **params)
 575
 576
 577 @Command(
 578     "wpt-manifest-update",
 579     category="testing",
 580     description="Update web-platform-test manifests.",
 581     parser=create_parser_manifest_update,
 582     virtualenv_name="wpt",
 583 )
 584 def wpt_manifest_update(command_context, **params):
 585     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 586     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 587     logger = wpt_runner.setup_logging(**params)
 588     logger.warning(
 589         "The wpt manifest is now automatically updated, "
 590         "so running this command is usually unnecessary"
 591     )
 592     return 0 if wpt_runner.update_manifest(logger, **params) else 1
 593
 594
 595 @Command(
 596     "wpt-serve",
 597     category="testing",
 598     description="Run the wpt server",
 599     parser=create_parser_serve,
 600     virtualenv_name="wpt",
 601 )
 602 def wpt_serve(command_context, **params):
 603     import logging
 604
 605     logger = logging.getLogger("web-platform-tests")
 606     logger.addHandler(logging.StreamHandler(sys.stdout))
 607     wpt_serve = command_context._spawn(WebPlatformTestsServeRunner)
 608     return wpt_serve.run(**params)
 609
 610
 611 @Command(
 612     "wpt-metadata-summary",
 613     category="testing",
 614     description="Create a json summary of the wpt metadata",
 615     parser=create_parser_metadata_summary,
 616     virtualenv_name="wpt",
 617 )
 618 def wpt_summary(command_context, **params):
 619     import metasummary
 620
 621     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 622     return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
 623
 624
 625 @Command(
 626     "wpt-metadata-merge",
 627     category="testing",
 628     parser=create_parser_metadata_merge,
 629     virtualenv_name="wpt",
 630 )
 631 def wpt_meta_merge(command_context, **params):
 632     import metamerge
 633
 634     if params["dest"] is None:
 635         params["dest"] = params["current"]
 636     return metamerge.run(**params)
 637
 638
 639 @Command(
 640     "wpt-unittest",
 641     category="testing",
 642     description="Run the wpt tools and wptrunner unit tests",
 643     parser=create_parser_unittest,
 644     virtualenv_name="wpt",
 645 )
 646 def wpt_unittest(command_context, **params):
 647     runner = command_context._spawn(WebPlatformTestsUnittestRunner)
 648     return 0 if runner.run(**params) else 1
 649
 650
 651 @Command(
 652     "wpt-test-paths",
 653     category="testing",
 654     description="Get a mapping from test ids to files",
 655     parser=create_parser_testpaths,
 656     virtualenv_name="wpt",
 657 )
 658 def wpt_test_paths(command_context, **params):
 659     runner = command_context._spawn(WebPlatformTestsTestPathsRunner)
 660     runner.run(**params)
 661     return 0
 662
 663
 664 @Command(
 665     "wpt-fission-regressions",
 666     category="testing",
 667     description="Dump a list of fission-specific regressions",
 668     parser=create_parser_fission_regressions,
 669     virtualenv_name="wpt",
 670 )
 671 def wpt_fission_regressions(command_context, **params):
 672     runner = command_context._spawn(WebPlatformTestsFissionRegressionsRunner)
 673     runner.run(**params)
 674     return 0
 675
 676
 677 @Command(
 678     "wpt-fetch-logs",
 679     category="testing",
 680     description="Fetch wptreport.json logs from taskcluster",
 681     parser=create_parser_fetch_logs,
 682     virtualenv_name="wpt-interop",
 683 )
 684 def wpt_fetch_logs(command_context, **params):
 685     import interop
 686
 687     interop.fetch_logs(**params)
 688     return 0
 689
 690
 691 @Command(
 692     "wpt-interop-score",
 693     category="testing",
 694     description="Score a run according to Interop 2023",
 695     parser=create_parser_interop_score,
 696     virtualenv_name="wpt-interop",
 697 )
 698 def wpt_interop_score(command_context, **params):
 699     import interop
 700
 701     interop.score_runs(**params)
 702     return 0