testing/web-platform/fissionregressions.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import os
   8 import re
   9 import sys
  10
  11 from mozlog import commandline
  12
  13 run_infos = {
  14     "linux-opt": {
  15         "os": "linux",
  16         "processor": "x86_64",
  17         "version": "Ubuntu 18.04",
  18         "os_version": "18.04",
  19         "bits": 64,
  20         "has_sandbox": True,
  21         "automation": True,
  22         "linux_distro": "Ubuntu",
  23         "apple_silicon": False,
  24         "appname": "firefox",
  25         "artifact": False,
  26         "asan": False,
  27         "bin_suffix": "",
  28         "buildapp": "browser",
  29         "buildtype_guess": "pgo",
  30         "cc_type": "clang",
  31         "ccov": False,
  32         "crashreporter": True,
  33         "datareporting": True,
  34         "debug": False,
  35         "devedition": False,
  36         "early_beta_or_earlier": True,
  37         "healthreport": True,
  38         "nightly_build": True,
  39         "normandy": True,
  40         "official": True,
  41         "pgo": True,
  42         "platform_guess": "linux64",
  43         "release_or_beta": False,
  44         "require_signing": False,
  45         "stylo": True,
  46         "sync": True,
  47         "telemetry": False,
  48         "tests_enabled": True,
  49         "toolkit": "gtk",
  50         "tsan": False,
  51         "ubsan": False,
  52         "updater": True,
  53         "python_version": 3,
  54         "product": "firefox",
  55         "verify": False,
  56         "wasm": True,
  57         "e10s": True,
  58         "headless": False,
  59         "fission": True,
  60         "sessionHistoryInParent": True,
  61         "swgl": False,
  62         "privateBrowsing": False,
  63         "win11_2009": False,
  64         "domstreams": True,
  65         "isolated_process": False,
  66         "display": "x11",
  67     },
  68     "linux-debug": {
  69         "os": "linux",
  70         "processor": "x86_64",
  71         "version": "Ubuntu 18.04",
  72         "os_version": "18.04",
  73         "bits": 64,
  74         "has_sandbox": True,
  75         "automation": True,
  76         "linux_distro": "Ubuntu",
  77         "apple_silicon": False,
  78         "appname": "firefox",
  79         "artifact": False,
  80         "asan": False,
  81         "bin_suffix": "",
  82         "buildapp": "browser",
  83         "buildtype_guess": "debug",
  84         "cc_type": "clang",
  85         "ccov": False,
  86         "crashreporter": True,
  87         "datareporting": True,
  88         "debug": True,
  89         "devedition": False,
  90         "early_beta_or_earlier": True,
  91         "healthreport": True,
  92         "nightly_build": True,
  93         "normandy": True,
  94         "official": True,
  95         "pgo": False,
  96         "platform_guess": "linux64",
  97         "release_or_beta": False,
  98         "require_signing": False,
  99         "stylo": True,
 100         "sync": True,
 101         "telemetry": False,
 102         "tests_enabled": True,
 103         "toolkit": "gtk",
 104         "tsan": False,
 105         "ubsan": False,
 106         "updater": True,
 107         "python_version": 3,
 108         "product": "firefox",
 109         "verify": False,
 110         "wasm": True,
 111         "e10s": True,
 112         "headless": False,
 113         "fission": False,
 114         "sessionHistoryInParent": False,
 115         "swgl": False,
 116         "privateBrowsing": False,
 117         "win11_2009": False,
 118         "domstreams": True,
 119         "isolated_process": False,
 120         "display": "x11",
 121     },
 122     "win-opt": {
 123         "os": "win",
 124         "processor": "x86_64",
 125         "version": "10.0.17134",
 126         "os_version": "10.0",
 127         "bits": 64,
 128         "has_sandbox": True,
 129         "automation": True,
 130         "apple_silicon": False,
 131         "appname": "firefox",
 132         "artifact": False,
 133         "asan": False,
 134         "bin_suffix": ".exe",
 135         "buildapp": "browser",
 136         "buildtype_guess": "pgo",
 137         "cc_type": "clang-cl",
 138         "ccov": False,
 139         "crashreporter": True,
 140         "datareporting": True,
 141         "debug": False,
 142         "devedition": False,
 143         "early_beta_or_earlier": True,
 144         "healthreport": True,
 145         "nightly_build": True,
 146         "normandy": True,
 147         "official": True,
 148         "pgo": True,
 149         "platform_guess": "win64",
 150         "release_or_beta": False,
 151         "require_signing": False,
 152         "stylo": True,
 153         "sync": True,
 154         "telemetry": False,
 155         "tests_enabled": True,
 156         "toolkit": "windows",
 157         "tsan": False,
 158         "ubsan": False,
 159         "updater": True,
 160         "python_version": 3,
 161         "product": "firefox",
 162         "verify": False,
 163         "wasm": True,
 164         "e10s": True,
 165         "headless": False,
 166         "fission": False,
 167         "sessionHistoryInParent": False,
 168         "swgl": False,
 169         "privateBrowsing": False,
 170         "win11_2009": False,
 171         "domstreams": True,
 172         "isolated_process": False,
 173         "display": None,
 174     },
 175 }
 176
 177
 178 # RE that checks for anything containing a three+ digit number
 179 maybe_bug_re = re.compile(r".*\d\d\d+")
 180
 181
 182 def get_parser():
 183     parser = argparse.ArgumentParser()
 184     parser.add_argument(
 185         "--all-json", type=os.path.abspath, help="Path to write json output to"
 186     )
 187     parser.add_argument(
 188         "--untriaged",
 189         type=os.path.abspath,
 190         help="Path to write list of regressions with no associated bug",
 191     )
 192     parser.add_argument(
 193         "--platform",
 194         dest="platforms",
 195         action="append",
 196         choices=list(run_infos.keys()),
 197         help="Configurations to compute fission changes for",
 198     )
 199     commandline.add_logging_group(parser)
 200     return parser
 201
 202
 203 def allowed_results(test, subtest=None):
 204     return test.expected(subtest), test.known_intermittent(subtest)
 205
 206
 207 def is_worse(baseline_result, new_result):
 208     if new_result == baseline_result:
 209         return False
 210
 211     if new_result in ("PASS", "OK"):
 212         return False
 213
 214     if baseline_result in ("PASS", "OK"):
 215         return True
 216
 217     # A crash -> not crash isn't a regression
 218     if baseline_result == "CRASH":
 219         return False
 220
 221     return True
 222
 223
 224 def is_regression(baseline_result, new_result):
 225     if baseline_result == new_result:
 226         return False
 227
 228     baseline_expected, baseline_intermittent = baseline_result
 229     new_expected, new_intermittent = new_result
 230
 231     baseline_all = {baseline_expected} | set(baseline_intermittent)
 232     new_all = {new_expected} | set(new_intermittent)
 233
 234     if baseline_all == new_all:
 235         return False
 236
 237     if not baseline_intermittent and not new_intermittent:
 238         return is_worse(baseline_expected, new_expected)
 239
 240     # If it was intermittent and isn't now, check if the new result is
 241     # worse than any of the previous results so that [PASS, FAIL] -> FAIL
 242     # looks like a regression
 243     if baseline_intermittent and not new_intermittent:
 244         return any(is_worse(result, new_expected) for result in baseline_all)
 245
 246     # If it was a perma and is now intermittent, check if any new result is
 247     # worse than the previous result.
 248     if not baseline_intermittent and new_intermittent:
 249         return any(is_worse(baseline_expected, result) for result in new_all)
 250
 251     # If it was an intermittent and is still an intermittent
 252     # check if any new result not in the old results is worse than
 253     # any old result
 254     new_results = new_all - baseline_all
 255     return any(
 256         is_worse(baseline_result, new_result)
 257         for new_result in new_results
 258         for baseline_result in baseline_all
 259     )
 260
 261
 262 def get_meta_prop(test, subtest, name):
 263     for meta in test.itermeta(subtest):
 264         try:
 265             value = meta.get(name)
 266         except KeyError:
 267             pass
 268         else:
 269             return value
 270     return None
 271
 272
 273 def include_result(result):
 274     if result.disabled or result.regressions:
 275         return True
 276
 277     if isinstance(result, TestResult):
 278         for subtest_result in result.subtest_results.values():
 279             if subtest_result.disabled or subtest_result.regressions:
 280                 return True
 281
 282     return False
 283
 284
 285 class Result:
 286     def __init__(self):
 287         self.bugs = set()
 288         self.disabled = set()
 289         self.regressions = {}
 290
 291     def add_regression(self, platform, baseline_results, fission_results):
 292         self.regressions[platform] = {
 293             "baseline": [baseline_results[0]] + baseline_results[1],
 294             "fission": [fission_results[0]] + fission_results[1],
 295         }
 296
 297     def to_json(self):
 298         raise NotImplementedError
 299
 300     def is_triaged(self):
 301         raise NotImplementedError
 302
 303
 304 class TestResult(Result):
 305     def __init__(self):
 306         super().__init__()
 307         self.subtest_results = {}
 308
 309     def add_subtest(self, name):
 310         self.subtest_results[name] = SubtestResult(self)
 311
 312     def to_json(self):
 313         rv = {}
 314         include_subtests = {
 315             name: item.to_json()
 316             for name, item in self.subtest_results.items()
 317             if include_result(item)
 318         }
 319         if include_subtests:
 320             rv["subtest_results"] = include_subtests
 321         if self.regressions:
 322             rv["regressions"] = self.regressions
 323         if self.disabled:
 324             rv["disabled"] = list(self.disabled)
 325         if self.bugs:
 326             rv["bugs"] = list(self.bugs)
 327         return rv
 328
 329     def is_triaged(self):
 330         return bool(self.bugs) or (
 331             not self.regressions
 332             and all(
 333                 subtest_result.is_triaged()
 334                 for subtest_result in self.subtest_results.values()
 335             )
 336         )
 337
 338
 339 class SubtestResult(Result):
 340     def __init__(self, parent):
 341         super().__init__()
 342         self.parent = parent
 343
 344     def to_json(self):
 345         rv = {}
 346         if self.regressions:
 347             rv["regressions"] = self.regressions
 348         if self.disabled:
 349             rv["disabled"] = list(self.disabled)
 350         bugs = self.bugs - self.parent.bugs
 351         if bugs:
 352             rv["bugs"] = list(bugs)
 353         return rv
 354
 355     def is_triaged(self):
 356         return bool(not self.regressions or self.parent.bugs or self.bugs)
 357
 358
 359 def run(logger, src_root, obj_root, **kwargs):
 360     commandline.setup_logging(
 361         logger, {key: value for key, value in kwargs.items() if key.startswith("log_")}
 362     )
 363
 364     import manifestupdate
 365
 366     sys.path.insert(
 367         0,
 368         os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 369     )
 370     from wptrunner import testloader, wpttest
 371
 372     logger.info("Loading test manifest")
 373     test_manifests = manifestupdate.run(src_root, obj_root, logger)
 374
 375     test_results = {}
 376
 377     platforms = kwargs["platforms"]
 378     if platforms is None:
 379         platforms = run_infos.keys()
 380
 381     for platform in platforms:
 382         platform_run_info = run_infos[platform]
 383         run_info_baseline = platform_run_info.copy()
 384         run_info_baseline["fission"] = False
 385
 386         tests = {}
 387
 388         for kind in ("baseline", "fission"):
 389             logger.info("Loading tests %s %s" % (platform, kind))
 390             run_info = platform_run_info.copy()
 391             run_info["fission"] = kind == "fission"
 392
 393             subsuites = testloader.load_subsuites(logger, run_info, None, set())
 394             test_loader = testloader.TestLoader(
 395                 test_manifests,
 396                 wpttest.enabled_tests,
 397                 run_info,
 398                 subsuites=subsuites,
 399                 manifest_filters=[],
 400             )
 401             tests[kind] = {
 402                 test.id: test
 403                 for _, _, test in test_loader.iter_tests(
 404                     run_info, test_loader.manifest_filters
 405                 )
 406                 if test._test_metadata is not None
 407             }
 408
 409         for test_id, baseline_test in tests["baseline"].items():
 410             fission_test = tests["fission"][test_id]
 411
 412             if test_id not in test_results:
 413                 test_results[test_id] = TestResult()
 414
 415             test_result = test_results[test_id]
 416
 417             baseline_bug = get_meta_prop(baseline_test, None, "bug")
 418             fission_bug = get_meta_prop(fission_test, None, "bug")
 419             if fission_bug and fission_bug != baseline_bug:
 420                 test_result.bugs.add(fission_bug)
 421
 422             if fission_test.disabled() and not baseline_test.disabled():
 423                 test_result.disabled.add(platform)
 424                 reason = get_meta_prop(fission_test, None, "disabled")
 425                 if reason and maybe_bug_re.match(reason):
 426                     test_result.bugs.add(reason)
 427
 428             baseline_results = allowed_results(baseline_test)
 429             fission_results = allowed_results(fission_test)
 430             result_is_regression = is_regression(baseline_results, fission_results)
 431
 432             if baseline_results != fission_results:
 433                 logger.debug(
 434                     "  %s %s %s %s"
 435                     % (test_id, baseline_results, fission_results, result_is_regression)
 436                 )
 437
 438             if result_is_regression:
 439                 test_result.add_regression(platform, baseline_results, fission_results)
 440
 441             for (
 442                 name,
 443                 baseline_subtest_meta,
 444             ) in baseline_test._test_metadata.subtests.items():
 445                 fission_subtest_meta = baseline_test._test_metadata.subtests[name]
 446                 if name not in test_result.subtest_results:
 447                     test_result.add_subtest(name)
 448
 449                 subtest_result = test_result.subtest_results[name]
 450
 451                 baseline_bug = get_meta_prop(baseline_test, name, "bug")
 452                 fission_bug = get_meta_prop(fission_test, name, "bug")
 453                 if fission_bug and fission_bug != baseline_bug:
 454                     subtest_result.bugs.add(fission_bug)
 455
 456                 if bool(fission_subtest_meta.disabled) and not bool(
 457                     baseline_subtest_meta.disabled
 458                 ):
 459                     subtest_result.disabled.add(platform)
 460                     if maybe_bug_re.match(fission_subtest_meta.disabled):
 461                         subtest_result.bugs.add(fission_subtest_meta.disabled)
 462
 463                 baseline_results = allowed_results(baseline_test, name)
 464                 fission_results = allowed_results(fission_test, name)
 465
 466                 result_is_regression = is_regression(baseline_results, fission_results)
 467
 468                 if baseline_results != fission_results:
 469                     logger.debug(
 470                         "    %s %s %s %s %s"
 471                         % (
 472                             test_id,
 473                             name,
 474                             baseline_results,
 475                             fission_results,
 476                             result_is_regression,
 477                         )
 478                     )
 479
 480                 if result_is_regression:
 481                     subtest_result.add_regression(
 482                         platform, baseline_results, fission_results
 483                     )
 484
 485     test_results = {
 486         test_id: result
 487         for test_id, result in test_results.items()
 488         if include_result(result)
 489     }
 490
 491     if kwargs["all_json"] is not None:
 492         write_all(test_results, kwargs["all_json"])
 493
 494     if kwargs["untriaged"] is not None:
 495         write_untriaged(test_results, kwargs["untriaged"])
 496
 497
 498 def write_all(test_results, path):
 499     json_data = {test_id: result.to_json() for test_id, result in test_results.items()}
 500
 501     dir_name = os.path.dirname(path)
 502     if not os.path.exists(dir_name):
 503         os.makedirs(dir_name)
 504
 505     with open(path, "w") as f:
 506         json.dump(json_data, f, indent=2)
 507
 508
 509 def write_untriaged(test_results, path):
 510     dir_name = os.path.dirname(path)
 511     if not os.path.exists(dir_name):
 512         os.makedirs(dir_name)
 513
 514     data = sorted(
 515         (test_id, result)
 516         for test_id, result in test_results.items()
 517         if not result.is_triaged()
 518     )
 519
 520     with open(path, "w") as f:
 521         for test_id, result in data:
 522             f.write(test_id + "\n")
 523             for name, subtest_result in sorted(result.subtest_results.items()):
 524                 if not subtest_result.is_triaged():
 525                     f.write("    %s\n" % name)