testing/webcompat/runner.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import errno
   6 import os
   7 import shutil
   8 import tempfile
   9
  10 import pytest
  11
  12 GVE = "org.mozilla.geckoview_example"
  13
  14
  15 def run(
  16     logger,
  17     path,
  18     webdriver_binary,
  19     webdriver_port,
  20     webdriver_ws_port,
  21     browser_binary=None,
  22     device_serial=None,
  23     package_name=None,
  24     environ=None,
  25     bug=None,
  26     debug=False,
  27     interventions=None,
  28     shims=None,
  29     config=None,
  30     headless=False,
  31     addon=None,
  32     do2fa=False,
  33 ):
  34     """"""
  35     old_environ = os.environ.copy()
  36     try:
  37         with TemporaryDirectory() as cache:
  38             if environ:
  39                 os.environ.update(environ)
  40
  41             config_plugin = WDConfig()
  42             result_recorder = ResultRecorder(logger)
  43
  44             args = [
  45                 "--strict",  # turn warnings into errors
  46                 "-vv",  # show each individual subtest and full failure logs
  47                 "--capture",
  48                 "no",  # enable stdout/stderr from tests
  49                 "--basetemp",
  50                 cache,  # temporary directory
  51                 "--showlocals",  # display contents of variables in local scope
  52                 "-p",
  53                 "no:mozlog",  # use the WPT result recorder
  54                 "--disable-warnings",
  55                 "-rfEs",
  56                 "-p",
  57                 "no:cacheprovider",  # disable state preservation across invocations
  58                 "-o=console_output_style=classic",  # disable test progress bar
  59                 "--browser",
  60                 "firefox",
  61                 "--webdriver-binary",
  62                 webdriver_binary,
  63                 "--webdriver-port",
  64                 webdriver_port,
  65                 "--webdriver-ws-port",
  66                 webdriver_ws_port,
  67             ]
  68
  69             if debug:
  70                 args.append("--pdb")
  71
  72             if headless:
  73                 args.append("--headless")
  74
  75             if browser_binary:
  76                 args.append("--browser-binary")
  77                 args.append(browser_binary)
  78
  79             if device_serial:
  80                 args.append("--device-serial")
  81                 args.append(device_serial)
  82
  83             if package_name:
  84                 args.append("--package-name")
  85                 args.append(package_name)
  86
  87             if addon:
  88                 args.append("--addon")
  89                 args.append(addon)
  90
  91             if bug:
  92                 args.append("--bug")
  93                 args.append(bug)
  94
  95             if do2fa:
  96                 args.append("--do2fa")
  97
  98             if config:
  99                 args.append("--config")
 100                 args.append(config)
 101
 102             if interventions is not None and shims is not None:
 103                 raise ValueError(
 104                     "Must provide only one of interventions or shims argument"
 105                 )
 106             elif interventions is None and shims is None:
 107                 raise ValueError(
 108                     "Must provide either an interventions or shims argument"
 109                 )
 110
 111             name = "webcompat-interventions"
 112             if interventions == "enabled":
 113                 args.extend(["-m", "with_interventions"])
 114             elif interventions == "disabled":
 115                 args.extend(["-m", "without_interventions"])
 116             elif interventions is not None:
 117                 raise ValueError(f"Invalid value for interventions {interventions}")
 118             if shims == "enabled":
 119                 args.extend(["-m", "with_shims"])
 120                 name = "smartblock-shims"
 121             elif shims == "disabled":
 122                 args.extend(["-m", "without_shims"])
 123                 name = "smartblock-shims"
 124             elif shims is not None:
 125                 raise ValueError(f"Invalid value for shims {shims}")
 126             else:
 127                 name = "smartblock-shims"
 128
 129             if bug is not None:
 130                 args.extend(["-k", bug])
 131
 132             args.append(path)
 133             try:
 134                 logger.suite_start([], name=name)
 135                 pytest.main(args, plugins=[config_plugin, result_recorder])
 136             except Exception as e:
 137                 logger.critical(str(e))
 138             finally:
 139                 logger.suite_end()
 140
 141     finally:
 142         os.environ = old_environ
 143
 144
 145 class WDConfig:
 146     def pytest_addoption(self, parser):
 147         parser.addoption(
 148             "--browser-binary", action="store", help="Path to browser binary"
 149         )
 150         parser.addoption(
 151             "--webdriver-binary", action="store", help="Path to webdriver binary"
 152         )
 153         parser.addoption(
 154             "--webdriver-port",
 155             action="store",
 156             default="4444",
 157             help="Port on which to run WebDriver",
 158         )
 159         parser.addoption(
 160             "--webdriver-ws-port",
 161             action="store",
 162             default="9222",
 163             help="Port on which to run WebDriver BiDi websocket",
 164         )
 165         parser.addoption(
 166             "--browser", action="store", choices=["firefox"], help="Name of the browser"
 167         )
 168         parser.addoption("--bug", action="store", help="Bug number to run tests for")
 169         parser.addoption(
 170             "--do2fa",
 171             action="store_true",
 172             default=False,
 173             help="Do two-factor auth live in supporting tests",
 174         )
 175         parser.addoption(
 176             "--config",
 177             action="store",
 178             help="Path to JSON file containing logins and other settings",
 179         )
 180         parser.addoption(
 181             "--addon",
 182             action="store",
 183             help="Path to the webcompat addon XPI to use",
 184         )
 185         parser.addoption(
 186             "--device-serial",
 187             action="store",
 188             help="Emulator device serial number",
 189         )
 190         parser.addoption(
 191             "--package-name",
 192             action="store",
 193             default=GVE,
 194             help="Android package to run/connect to",
 195         )
 196         parser.addoption(
 197             "--headless", action="store_true", help="Run browser in headless mode"
 198         )
 199
 200
 201 class ResultRecorder(object):
 202     def __init__(self, logger):
 203         self.logger = logger
 204
 205     def pytest_runtest_logreport(self, report):
 206         if report.passed and report.when == "call":
 207             self.record_pass(report)
 208         elif report.failed:
 209             if report.when != "call":
 210                 self.record_error(report)
 211             else:
 212                 self.record_fail(report)
 213         elif report.skipped:
 214             self.record_skip(report)
 215
 216     def record_pass(self, report):
 217         self.record(report.nodeid, "PASS")
 218
 219     def record_fail(self, report):
 220         # pytest outputs the stacktrace followed by an error message prefixed
 221         # with "E   ", e.g.
 222         #
 223         #        def test_example():
 224         #  >         assert "fuu" in "foobar"
 225         #  > E       AssertionError: assert 'fuu' in 'foobar'
 226         message = ""
 227         for line in report.longreprtext.splitlines():
 228             if line.startswith("E   "):
 229                 message = line[1:].strip()
 230                 break
 231
 232         self.record(report.nodeid, "FAIL", message=message, stack=report.longrepr)
 233
 234     def record_error(self, report):
 235         # error in setup/teardown
 236         if report.when != "call":
 237             message = "%s error" % report.when
 238         self.record(report.nodeid, "ERROR", message, report.longrepr)
 239
 240     def record_skip(self, report):
 241         self.record(report.nodeid, "SKIP")
 242
 243     def record(self, test, status, message=None, stack=None):
 244         if stack is not None:
 245             stack = str(stack)
 246         self.logger.test_start(test)
 247         self.logger.test_end(
 248             test=test, status=status, expected="PASS", message=message, stack=stack
 249         )
 250
 251
 252 class TemporaryDirectory(object):
 253     def __enter__(self):
 254         self.path = tempfile.mkdtemp(prefix="pytest-")
 255         return self.path
 256
 257     def __exit__(self, *args):
 258         try:
 259             shutil.rmtree(self.path)
 260         except OSError as e:
 261             # no such file or directory
 262             if e.errno != errno.ENOENT:
 263                 raise