From 528dbee04cc6c6b685737a20bcb7b6d1e89a2ee3 Mon Sep 17 00:00:00 2001 From: Greg Mierzwinski Date: Tue, 24 Oct 2023 13:05:49 +0000 Subject: [PATCH] Bug 1832059 - Add a test runner for functional tests to mozperftest. r=perftest-reviewers,kshampur This patch adds a FunctionalTestRunner class that can be used by all the layers to run `./mach test` which lets us easily run all functional tests locally. It will be used for running mochitest tests, and in the future, it may replace parts of the xpcshell layer. The runner can only be used locally, and only needs a test path provided. It will then run all the manifests that it finds that test in (but only running that single test). At the same time, it'll parse the logs produced by `./mach test` with FunctionalTestProcessor to find all the metrics that were output. However, it does not parse them into a JSON format, that's left for the layers to handle. Differential Revision: https://phabricator.services.mozilla.com/D190993 --- .../mozperftest/test/functionaltestrunner.py | 82 ++++++++++++++++++++++ .../mozperftest/tests/test_functionaltestrunner.py | 57 +++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 python/mozperftest/mozperftest/test/functionaltestrunner.py create mode 100644 python/mozperftest/mozperftest/tests/test_functionaltestrunner.py diff --git a/python/mozperftest/mozperftest/test/functionaltestrunner.py b/python/mozperftest/mozperftest/test/functionaltestrunner.py new file mode 100644 index 000000000000..e099c6b9763f --- /dev/null +++ b/python/mozperftest/mozperftest/test/functionaltestrunner.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pathlib +import sys + +import mozlog + +from mozperftest.utils import load_class_from_path + + +class FunctionalTestProcessor(mozlog.handlers.StreamHandler): + """Used for capturing the perfMetrics output from a `mach test` run.""" + + def __init__(self, *args, **kwargs): + self._match = [] + super(FunctionalTestProcessor, self).__init__(*args, **kwargs) + + def __call__(self, data): + formatted = self.formatter(data) + if formatted is not None and "perfMetrics" in formatted: + self.match.append(formatted) + + @property + def match(self): + return self._match + + +class FunctionalTestRunner: + def test(command_context, what, extra_args, **log_args): + """Run tests from names or paths. + + Based on the `mach test` command in testing/mach_commands.py. It uses + the same logic but with less features for logging, and a custom log + handler to capture the perfMetrics output. + """ + from mozlog.commandline import setup_logging + from moztest.resolve import TestResolver + + resolver = command_context._spawn(TestResolver) + run_suites, run_tests = resolver.resolve_metadata(what) + + if not run_suites and not run_tests: + print( + "Could not find the requested test. Ensure that it works with `./mach test`." + ) + return 1, None + + # Create shared logger + setup_logging( + "mach-test", + log_args, + {"mach": sys.stdout}, + {"level": "info", "verbose": True, "compact": False}, + ) + + # Make a custom handler to capture the perfMetrics log message + log_processor = FunctionalTestProcessor( + stream=sys.stdout, + formatter=mozlog.formatters.MachFormatter( + verbose=True, disable_colors=False + ), + ) + + # Setup the runner + machtestrunner = load_class_from_path( + "MachTestRunner", + pathlib.Path(command_context.topsrcdir, "testing/mach_commands.py"), + ) + command_context._mach_context.settings = { + "test": { + "level": "info", + "verbose": True, + "compact": False, + "format": "mach", + }, + } + log_args["custom_handler"] = log_processor + + # Run the test + status = machtestrunner.test(command_context, what, extra_args, **log_args) + return status, log_processor diff --git a/python/mozperftest/mozperftest/tests/test_functionaltestrunner.py b/python/mozperftest/mozperftest/tests/test_functionaltestrunner.py new file mode 100644 index 000000000000..20f8362d60a0 --- /dev/null +++ b/python/mozperftest/mozperftest/tests/test_functionaltestrunner.py @@ -0,0 +1,57 @@ +import sys +from unittest import mock + +from mozperftest.test.functionaltestrunner import ( + FunctionalTestProcessor, + FunctionalTestRunner, +) + + +def test_functionaltestrunner_pass(): + with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path" + ) as load_class_path_mock, mock.patch( + "mozperftest.test.functionaltestrunner.FunctionalTestProcessor" + ), mock.patch( + "mozperftest.test.functionaltestrunner.mozlog" + ): + test_mock = mock.MagicMock() + test_mock.test.return_value = 0 + load_class_path_mock.return_value = test_mock + + mach_cmd = mock.MagicMock() + test_resolver_mock.resolve_metadata.return_value = (1, 1) + mach_cmd._spawn.return_value = test_resolver_mock + + status, _ = FunctionalTestRunner.test(mach_cmd, [], []) + + assert status == 0 + + +def test_functionaltestrunner_missing_test_failure(): + with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock: + mach_cmd = mock.MagicMock() + test_resolver_mock.resolve_metadata.return_value = (0, 0) + mach_cmd._spawn.return_value = test_resolver_mock + status, _ = FunctionalTestRunner.test(mach_cmd, [], []) + assert status == 1 + + +def test_functionaltestrunner_perfmetrics_parsing(): + formatter_mock = mock.MagicMock() + formatter_mock.return_value = "perfMetrics | fake-data" + + log_processor = FunctionalTestProcessor(stream=sys.stdout, formatter=formatter_mock) + log_processor("") + + assert len(log_processor.match) == 1 + + +def test_functionaltestrunner_perfmetrics_missing(): + formatter_mock = mock.MagicMock() + formatter_mock.return_value = "perfmetrics | fake-data" + + log_processor = FunctionalTestProcessor(stream=sys.stdout, formatter=formatter_mock) + log_processor("") + + assert len(log_processor.match) == 0 -- 2.11.4.GIT