From d48a3983f642ae8a6198a860c914c45204874b14 Mon Sep 17 00:00:00 2001 From: Sam Sneddon Date: Mon, 20 Mar 2023 18:03:21 +0000 Subject: [PATCH] Bug 1807109 [wpt PR 37653] - Initial support for running tests against WebKitTestRunner, a=testonly Automatic update from web-platform-tests Initial support for running tests against WebKitTestRunner (#37653) -- wpt-commits: 0f8899ecf3856487e50522654fd4cc0f4dc6b7c3 wpt-pr: 37653 --- testing/web-platform/tests/tools/wpt/browser.py | 26 ++ testing/web-platform/tests/tools/wpt/run.py | 14 +- .../tools/wptrunner/wptrunner/browsers/__init__.py | 1 + .../tools/wptrunner/wptrunner/browsers/wktr.py | 239 ++++++++++++++++++ .../wptrunner/wptrunner/executors/executorwktr.py | 268 +++++++++++++++++++++ .../wptrunner/wptrunner/testharnessreport-wktr.js | 23 ++ 6 files changed, 570 insertions(+), 1 deletion(-) create mode 100644 testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/wktr.py create mode 100644 testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwktr.py create mode 100644 testing/web-platform/tests/tools/wptrunner/wptrunner/testharnessreport-wktr.js diff --git a/testing/web-platform/tests/tools/wpt/browser.py b/testing/web-platform/tests/tools/wpt/browser.py index 66796a8968d9..7d789f6a71c7 100644 --- a/testing/web-platform/tests/tools/wpt/browser.py +++ b/testing/web-platform/tests/tools/wpt/browser.py @@ -1883,6 +1883,32 @@ class WebKit(Browser): return None +class WebKitTestRunner(Browser): + """Interface for WebKitTestRunner. + """ + + product = "wktr" + requirements = None + + def download(self, dest=None, channel=None, rename=None): + raise NotImplementedError + + def install(self, dest=None, channel=None): + raise NotImplementedError + + def install_webdriver(self, dest=None, channel=None, browser_binary=None): + raise NotImplementedError + + def find_binary(self, venv_path=None, channel=None): + return None + + def find_webdriver(self, venv_path=None, channel=None): + return None + + def version(self, binary=None, webdriver_binary=None): + return None + + class WebKitGTKMiniBrowser(WebKit): diff --git a/testing/web-platform/tests/tools/wpt/run.py b/testing/web-platform/tests/tools/wpt/run.py index feed5a5625d7..a40f60dfdade 100644 --- a/testing/web-platform/tests/tools/wpt/run.py +++ b/testing/web-platform/tests/tools/wpt/run.py @@ -110,7 +110,7 @@ otherwise install OpenSSL and ensure that it's on your $PATH.""") def check_environ(product): if product not in ("android_weblayer", "android_webview", "chrome", "chrome_android", "chrome_ios", "content_shell", - "firefox", "firefox_android", "servo"): + "firefox", "firefox_android", "servo", "wktr"): config_builder = serve.build_config(os.path.join(wpt_root, "config.json")) # Override the ports to avoid looking for free ports config_builder.ssl = {"type": "none"} @@ -688,6 +688,17 @@ class WebKit(BrowserSetup): pass +class WebKitTestRunner(BrowserSetup): + name = "wktr" + browser_cls = browser.WebKitTestRunner + + def install(self, channel=None): + raise NotImplementedError + + def setup_kwargs(self, kwargs): + pass + + class WebKitGTKMiniBrowser(BrowserSetup): name = "webkitgtk_minibrowser" browser_cls = browser.WebKitGTKMiniBrowser @@ -757,6 +768,7 @@ product_setup = { "sauce": Sauce, "opera": Opera, "webkit": WebKit, + "wktr": WebKitTestRunner, "webkitgtk_minibrowser": WebKitGTKMiniBrowser, "epiphany": Epiphany, } diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py index b2a53ca23a7b..9724bb957b5e 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py @@ -42,4 +42,5 @@ product_list = ["android_weblayer", "opera", "webkit", "webkitgtk_minibrowser", + "wktr", "epiphany"] diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/wktr.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/wktr.py new file mode 100644 index 000000000000..8d429f357b33 --- /dev/null +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/wktr.py @@ -0,0 +1,239 @@ +# mypy: allow-untyped-defs + +import gc +import os +import sys +from multiprocessing import Queue +from subprocess import PIPE +from threading import Thread +from mozprocess import ProcessHandlerMixin + +from .base import Browser, ExecutorBrowser +from .base import get_timeout_multiplier # noqa: F401 +from ..executors import executor_kwargs as base_executor_kwargs +from ..executors.executorwktr import ( # noqa: F401 + WKTRCrashtestExecutor, + WKTRRefTestExecutor, + WKTRTestharnessExecutor, +) + + +__wptrunner__ = {"product": "WebKitTestRunner", + "check_args": "check_args", + "browser": "WKTRBrowser", + "executor": { + "crashtest": "WKTRCrashtestExecutor", + "reftest": "WKTRRefTestExecutor", + "testharness": "WKTRTestharnessExecutor", + }, + "browser_kwargs": "browser_kwargs", + "executor_kwargs": "executor_kwargs", + "env_extras": "env_extras", + "env_options": "env_options", + "update_properties": "update_properties", + "timeout_multiplier": "get_timeout_multiplier",} + + +def check_args(**kwargs): + pass + + +def browser_kwargs(logger, test_type, run_info_data, config, **kwargs): + args = list(kwargs["binary_args"]) + + args.append("--allow-any-certificate-for-allowed-hosts") + + for host in config.domains_set: + args.append('--allowed-host') + args.append(host) + + args.append("-") + + return {"binary": kwargs["binary"], + "binary_args": args} + + +def executor_kwargs(logger, test_type, test_environment, run_info_data, + **kwargs): + executor_kwargs = base_executor_kwargs(test_type, test_environment, run_info_data, + **kwargs) + return executor_kwargs + + +def env_extras(**kwargs): + return [] + + +def env_options(): + return {"server_host": "127.0.0.1", + "testharnessreport": "testharnessreport-wktr.js"} + + +def update_properties(): + return (["debug", "os", "processor"], {"os": ["version"], "processor": ["bits"]}) + + +class WKTRBrowser(Browser): + """Class that represents an instance of WebKitTestRunner. + + Upon startup, the stdout, stderr, and stdin pipes of the underlying WebKitTestRunner + process are connected to multiprocessing Queues so that the runner process can + interact with WebKitTestRunner through its protocol mode. + """ + + def __init__(self, logger, binary="WebKitTestRunner", binary_args=[], **kwargs): + super().__init__(logger) + + self._args = [binary] + binary_args + self._proc = None + + build_root_path = os.path.dirname(binary) + + def append_env(var, item, separator): + raw = os.environ.get(var) + old = raw.split(separator) if raw is not None else [] + return separator.join(old + [item]) + + env = {} + + if sys.platform.startswith("darwin"): + env["CA_DISABLE_GENERIC_SHADERS"] = "1" + env["__XPC_CA_DISABLE_GENERIC_SHADERS"] = "1" + + env["DYLD_LIBRARY_PATH"] = append_env( + "DYLD_LIBRARY_PATH", build_root_path, ":" + ) + env["__XPC_DYLD_LIBRARY_PATH"] = append_env( + "__XPC_DYLD_LIBRARY_PATH", build_root_path, ":" + ) + env["DYLD_FRAMEWORK_PATH"] = append_env( + "DYLD_FRAMEWORK_PATH", build_root_path, ":" + ) + env["__XPC_DYLD_FRAMEWORK_PATH"] = append_env( + "__XPC_DYLD_FRAMEWORK_PATH", build_root_path, ":" + ) + + env["SQLITE_EXEMPT_PATH_FROM_VNODE_GUARDS"] = "/" + env["__XPC_SQLITE_EXEMPT_PATH_FROM_VNODE_GUARDS"] = "/" + + self._extra_env = env + + def start(self, group_metadata, **kwargs): + self.logger.debug("Starting WebKitTestRunner: %s..." % self._args[0]) + #self.logger.debug(repr(self._args)) + + env = os.environ + env.update(self._extra_env) + + # Unfortunately we need to use the Process class directly because we do not + # want mozprocess to do any output handling at all. + self._proc = ProcessHandlerMixin.Process( + self._args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env + ) + if os.name == "posix": + self._proc.pgid = ProcessHandlerMixin._getpgid(self._proc.pid) + self._proc.detached_pid = None + + self._stdout_queue = Queue() + self._stderr_queue = Queue() + self._stdin_queue = Queue() + + self._stdout_reader = self._create_reader_thread(self._proc.stdout, self._stdout_queue) + self._stderr_reader = self._create_reader_thread(self._proc.stderr, self._stderr_queue) + self._stdin_writer = self._create_writer_thread(self._proc.stdin, self._stdin_queue) + + # WebKitTestRunner is likely still in the process of initializing. The actual waiting + # for the startup to finish is done in the WKTRProtocol. + self.logger.debug("WebKitTestRunner has been started.") + + def stop(self, force=False): + self.logger.debug("Stopping WebKitTestRunner...") + + if self.is_alive(): + kill_result = self._proc.kill(timeout=5) + # This makes sure any left-over child processes get killed. + # See http://bugzilla.mozilla.org/show_bug.cgi?id=1760080 + if force and kill_result != 0: + self._proc.kill(9, timeout=5) + + # We need to shut down these queues cleanly to avoid broken pipe error spam in the logs. + self._stdout_reader.join(2) + self._stderr_reader.join(2) + + self._stdin_queue.put(None) + self._stdin_writer.join(2) + + for thread in [self._stdout_reader, self._stderr_reader, self._stdin_writer]: + if thread.is_alive(): + self.logger.warning("WebKitTestRunner IO threads did not shut down gracefully.") + return False + + stopped = not self.is_alive() + if stopped: + self._proc = None + self._stdout_queue.close() + self.logger.debug("WebKitTestRunner has been stopped.") + + # We sometimes accumulate too many process-related objects, + # ultimately running out of OS file handles, via circular + # references to them, thus manually trigger a GC while stopping. + gc.collect() + else: + self.logger.warning("WebKitTestRunner failed to stop.") + + return stopped + + def is_alive(self): + return self._proc is not None and self._proc.poll() is None + + def pid(self): + return self._proc.pid if self._proc else None + + def executor_browser(self): + """This function returns the `ExecutorBrowser` object that is used by other + processes to interact with WebKitTestRunner. In our case, this consists of the three + multiprocessing Queues as well as an `io_stopped` event to signal when the + underlying pipes have reached EOF. + """ + return ExecutorBrowser, {"stdout_queue": self._stdout_queue, + "stderr_queue": self._stderr_queue, + "stdin_queue": self._stdin_queue} + + def check_crash(self, process, test): + return not self.is_alive() + + def _create_reader_thread(self, stream, queue): + """This creates (and starts) a background thread which reads lines from `stream` and + puts them into `queue` until `stream` reports EOF. + """ + def reader_thread(stream, queue): + while True: + line = stream.readline() + if not line: + break + + queue.put(line) + + queue.close() + queue.join_thread() + + result = Thread(target=reader_thread, args=(stream, queue), daemon=True) + result.start() + return result + + def _create_writer_thread(self, stream, queue): + """This creates (and starts) a background thread which gets items from `queue` and + writes them into `stream` until it encounters a None item in the queue. + """ + def writer_thread(stream, queue): + while True: + line = queue.get() + if not line: + break + + stream.write(line) + stream.flush() + + result = Thread(target=writer_thread, args=(stream, queue), daemon=True) + result.start() + return result diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwktr.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwktr.py new file mode 100644 index 000000000000..ab94000fa209 --- /dev/null +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwktr.py @@ -0,0 +1,268 @@ +# mypy: allow-untyped-defs + +from .base import RefTestExecutor, RefTestImplementation, CrashtestExecutor, TestharnessExecutor +from .protocol import Protocol, ProtocolPart +from time import time +from queue import Empty +from base64 import b64encode +import json + + +class CrashError(BaseException): + pass + + +def _read_line(io_queue, deadline=None, encoding=None, errors="strict", raise_crash=True, logger=None): + """Reads a single line from the io queue. The read must succeed before `deadline` or + a TimeoutError is raised. The line is returned as a bytestring or optionally with the + specified `encoding`. If `raise_crash` is set, a CrashError is raised if the line + happens to be a crash message. + """ + current_time = time() + + if deadline and current_time > deadline: + raise TimeoutError() + + try: + line = io_queue.get(True, deadline - current_time if deadline else None) + if raise_crash and line.startswith(b"#CRASHED"): + raise CrashError() + except Empty: + logger.debug(f"got empty line with {time() - deadline} remaining") + raise TimeoutError() + + return line.decode(encoding, errors) if encoding else line + + +class WKTRTestPart(ProtocolPart): + """This protocol part is responsible for running tests via WebKitTestRunner's protocol mode. + """ + name = "wktr_test" + eof_marker = '#EOF\n' # Marker sent by wktr after blocks. + + def __init__(self, parent): + super().__init__(parent) + self.stdout_queue = parent.browser.stdout_queue + self.stdin_queue = parent.browser.stdin_queue + + def do_test(self, command, timeout=None): + """Send a command to wktr and return the resulting outputs. + + A command consists of a URL to navigate to, followed by an optional options; see + https://github.com/WebKit/WebKit/blob/main/Tools/TestRunnerShared/TestCommand.cpp. + + """ + self._send_command(command + "'--timeout'%d" % (timeout * 1000)) + + deadline = time() + timeout if timeout else None + # The first block can also contain audio data but not in WPT. + text = self._read_block(deadline) + image = self._read_block(deadline) + + return text, image + + def _send_command(self, command): + """Sends a single `command`, i.e. a URL to open, to wktr. + """ + self.stdin_queue.put((command + "\n").encode("utf-8")) + + def _read_block(self, deadline=None): + """Tries to read a single block of content from stdout before the `deadline`. + """ + while True: + line = _read_line(self.stdout_queue, deadline, "latin-1", logger=self.logger).rstrip() + + if line == "Content-Type: text/plain": + return self._read_text_block(deadline) + + if line == "Content-Type: image/png": + return self._read_image_block(deadline) + + if line == "#EOF": + return None + + def _read_text_block(self, deadline=None): + """Tries to read a plain text block in utf-8 encoding before the `deadline`. + """ + result = "" + + while True: + line = _read_line(self.stdout_queue, deadline, "utf-8", "replace", False, logger=self.logger) + + if line.endswith(self.eof_marker): + result += line[:-len(self.eof_marker)] + break + elif line.endswith('#EOF\r\n'): + result += line[:-len('#EOF\r\n')] + self.logger.warning('Got a CRLF-terminated #EOF - this is a driver bug.') + break + + result += line + + return result + + def _read_image_block(self, deadline=None): + """Tries to read an image block (as a binary png) before the `deadline`. + """ + content_length_line = _read_line(self.stdout_queue, deadline, "utf-8", logger=self.logger) + assert content_length_line.startswith("Content-Length:") + content_length = int(content_length_line[15:]) + + result = bytearray() + + while True: + line = _read_line(self.stdout_queue, deadline, raise_crash=False, logger=self.logger) + excess = len(line) + len(result) - content_length + + if excess > 0: + # This is the line that contains the EOF marker. + assert excess == len(self.eof_marker) + result += line[:-excess] + break + + result += line + + return result + + +class WKTRErrorsPart(ProtocolPart): + """This protocol part is responsible for collecting the errors reported by wktr. + """ + name = "wktr_errors" + + def __init__(self, parent): + super().__init__(parent) + self.stderr_queue = parent.browser.stderr_queue + + def read_errors(self): + """Reads the entire content of the stderr queue as is available right now (no blocking). + """ + result = "" + + while not self.stderr_queue.empty(): + # There is no potential for race conditions here because this is the only place + # where we read from the stderr queue. + result += _read_line(self.stderr_queue, None, "utf-8", "replace", False, logger=self.logger) + + return result + + +class WKTRProtocol(Protocol): + implements = [WKTRTestPart, WKTRErrorsPart] + + def connect(self): + pass + + def after_connect(self): + pass + + def teardown(self): + # Close the queue properly to avoid broken pipe spam in the log. + self.browser.stdin_queue.close() + self.browser.stdin_queue.join_thread() + + def is_alive(self): + """Checks if wktr is alive by determining if the IO pipes are still + open. This does not guarantee that the process is responsive. + """ + return self.browser.io_stopped.is_set() + + +def _convert_exception(test, exception, errors): + """Converts our TimeoutError and CrashError exceptions into test results. + """ + if isinstance(exception, TimeoutError): + return (test.result_cls("EXTERNAL-TIMEOUT", errors), []) + if isinstance(exception, CrashError): + return (test.result_cls("CRASH", errors), []) + raise exception + + +class WKTRRefTestExecutor(RefTestExecutor): + def __init__(self, logger, browser, server_config, timeout_multiplier=1, screenshot_cache=None, + debug_info=None, reftest_screenshot="unexpected", **kwargs): + super().__init__(logger, browser, server_config, timeout_multiplier, screenshot_cache, + debug_info, reftest_screenshot, **kwargs) + self.implementation = RefTestImplementation(self) + self.protocol = WKTRProtocol(self, browser) + + def reset(self): + self.implementation.reset() + + def do_test(self, test): + try: + result = self.implementation.run_test(test) + self.protocol.wktr_errors.read_errors() + return self.convert_result(test, result) + except BaseException as exception: + return _convert_exception(test, exception, self.protocol.wktr_errors.read_errors()) + + def screenshot(self, test, viewport_size, dpi, page_ranges): + assert dpi is None + command = self.test_url(test) + command += "'--pixel-test'" + assert not self.is_print + _, image = self.protocol.wktr_test.do_test( + command, test.timeout * self.timeout_multiplier) + + if not image: + return False, ("ERROR", self.protocol.wktr_errors.read_errors()) + + return True, b64encode(image).decode() + + def wait(self): + return + + +class WKTRCrashtestExecutor(CrashtestExecutor): + def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None, + **kwargs): + super().__init__(logger, browser, server_config, timeout_multiplier, debug_info, **kwargs) + self.protocol = WKTRProtocol(self, browser) + + def do_test(self, test): + try: + _ = self.protocol.wktr_test.do_test(self.test_url(test), test.timeout * self.timeout_multiplier) + self.protocol.wktr_errors.read_errors() + return self.convert_result(test, {"status": "PASS", "message": None}) + except BaseException as exception: + return _convert_exception(test, exception, self.protocol.wktr_errors.read_errors()) + + def wait(self): + return + + +class WKTRTestharnessExecutor(TestharnessExecutor): + def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None, + **kwargs): + super().__init__(logger, browser, server_config, timeout_multiplier, debug_info, **kwargs) + self.protocol = WKTRProtocol(self, browser) + + def do_test(self, test): + try: + text, _ = self.protocol.wktr_test.do_test(self.test_url(test), + test.timeout * self.timeout_multiplier) + + errors = self.protocol.wktr_errors.read_errors() + if not text: + return (test.result_cls("ERROR", errors), []) + + output = None + output_prefix = "CONSOLE MESSAGE: WPTRUNNER OUTPUT:" + + for line in text.split("\n"): + if line.startswith(output_prefix): + if output is None: + output = line[len(output_prefix):] + else: + return (test.result_cls("ERROR", "multiple wptrunner outputs"), []) + + if output is None: + return (test.result_cls("ERROR", "no wptrunner output"), []) + + return self.convert_result(test, json.loads(output)) + except BaseException as exception: + return _convert_exception(test, exception, self.protocol.wktr_errors.read_errors()) + + def wait(self): + return diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/testharnessreport-wktr.js b/testing/web-platform/tests/tools/wptrunner/wptrunner/testharnessreport-wktr.js new file mode 100644 index 000000000000..b7d350a4262c --- /dev/null +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testharnessreport-wktr.js @@ -0,0 +1,23 @@ +var props = {output:%(output)d, debug: %(debug)s}; +var start_loc = document.createElement('a'); +start_loc.href = location.href; +setup(props); + +testRunner.dumpAsText(); +testRunner.waitUntilDone(); + +add_completion_callback(function (tests, harness_status) { + var id = decodeURIComponent(start_loc.pathname) + decodeURIComponent(start_loc.search) + decodeURIComponent(start_loc.hash); + var result_string = JSON.stringify([ + id, + harness_status.status, + harness_status.message, + harness_status.stack, + tests.map(function(t) { + return [t.name, t.status, t.message, t.stack] + }), + ]); + + console.log("WPTRUNNER OUTPUT:" + result_string); + testRunner.notifyDone(); +}); -- 2.11.4.GIT