testing/web-platform/fissionregressions.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import os
   8 import re
   9 import sys
  10
  11 from mozlog import commandline
  12
  13 run_infos = {
  14     "linux-opt": {
  15         "os": "linux",
  16         "processor": "x86_64",
  17         "version": "Ubuntu 18.04",
  18         "os_version": "18.04",
  19         "bits": 64,
  20         "has_sandbox": True,
  21         "automation": True,
  22         "linux_distro": "Ubuntu",
  23         "apple_silicon": False,
  24         "appname": "firefox",
  25         "artifact": False,
  26         "asan": False,
  27         "bin_suffix": "",
  28         "buildapp": "browser",
  29         "buildtype_guess": "pgo",
  30         "cc_type": "clang",
  31         "ccov": False,
  32         "crashreporter": True,
  33         "datareporting": True,
  34         "debug": False,
  35         "devedition": False,
  36         "early_beta_or_earlier": True,
  37         "healthreport": True,
  38         "nightly_build": True,
  39         "non_native_theme": True,
  40         "normandy": True,
  41         "official": True,
  42         "pgo": True,
  43         "platform_guess": "linux64",
  44         "release_or_beta": False,
  45         "require_signing": False,
  46         "stylo": True,
  47         "sync": True,
  48         "telemetry": False,
  49         "tests_enabled": True,
  50         "toolkit": "gtk",
  51         "tsan": False,
  52         "ubsan": False,
  53         "updater": True,
  54         "python_version": 3,
  55         "product": "firefox",
  56         "verify": False,
  57         "wasm": True,
  58         "e10s": True,
  59         "headless": False,
  60         "fission": True,
  61         "sessionHistoryInParent": True,
  62         "swgl": False,
  63         "win10_2004": False,
  64         "domstreams": True,
  65         "isolated_process": False,
  66     },
  67     "linux-debug": {
  68         "os": "linux",
  69         "processor": "x86_64",
  70         "version": "Ubuntu 18.04",
  71         "os_version": "18.04",
  72         "bits": 64,
  73         "has_sandbox": True,
  74         "automation": True,
  75         "linux_distro": "Ubuntu",
  76         "apple_silicon": False,
  77         "appname": "firefox",
  78         "artifact": False,
  79         "asan": False,
  80         "bin_suffix": "",
  81         "buildapp": "browser",
  82         "buildtype_guess": "debug",
  83         "cc_type": "clang",
  84         "ccov": False,
  85         "crashreporter": True,
  86         "datareporting": True,
  87         "debug": True,
  88         "devedition": False,
  89         "early_beta_or_earlier": True,
  90         "healthreport": True,
  91         "nightly_build": True,
  92         "non_native_theme": True,
  93         "normandy": True,
  94         "official": True,
  95         "pgo": False,
  96         "platform_guess": "linux64",
  97         "release_or_beta": False,
  98         "require_signing": False,
  99         "stylo": True,
 100         "sync": True,
 101         "telemetry": False,
 102         "tests_enabled": True,
 103         "toolkit": "gtk",
 104         "tsan": False,
 105         "ubsan": False,
 106         "updater": True,
 107         "python_version": 3,
 108         "product": "firefox",
 109         "verify": False,
 110         "wasm": True,
 111         "e10s": True,
 112         "headless": False,
 113         "fission": False,
 114         "sessionHistoryInParent": False,
 115         "swgl": False,
 116         "win10_2004": False,
 117         "domstreams": True,
 118         "isolated_process": False,
 119     },
 120     "win-opt": {
 121         "os": "win",
 122         "processor": "x86_64",
 123         "version": "10.0.17134",
 124         "os_version": "10.0",
 125         "bits": 64,
 126         "has_sandbox": True,
 127         "automation": True,
 128         "service_pack": "",
 129         "apple_silicon": False,
 130         "appname": "firefox",
 131         "artifact": False,
 132         "asan": False,
 133         "bin_suffix": ".exe",
 134         "buildapp": "browser",
 135         "buildtype_guess": "pgo",
 136         "cc_type": "clang-cl",
 137         "ccov": False,
 138         "crashreporter": True,
 139         "datareporting": True,
 140         "debug": False,
 141         "devedition": False,
 142         "early_beta_or_earlier": True,
 143         "healthreport": True,
 144         "nightly_build": True,
 145         "non_native_theme": False,
 146         "normandy": True,
 147         "official": True,
 148         "pgo": True,
 149         "platform_guess": "win64",
 150         "release_or_beta": False,
 151         "require_signing": False,
 152         "stylo": True,
 153         "sync": True,
 154         "telemetry": False,
 155         "tests_enabled": True,
 156         "toolkit": "windows",
 157         "tsan": False,
 158         "ubsan": False,
 159         "updater": True,
 160         "python_version": 3,
 161         "product": "firefox",
 162         "verify": False,
 163         "wasm": True,
 164         "e10s": True,
 165         "headless": False,
 166         "fission": False,
 167         "sessionHistoryInParent": False,
 168         "swgl": False,
 169         "win10_2004": False,
 170         "domstreams": True,
 171         "isolated_process": False,
 172     },
 173 }
 174
 175
 176 # RE that checks for anything containing a three+ digit number
 177 maybe_bug_re = re.compile(r".*\d\d\d+")
 178
 179
 180 def get_parser():
 181     parser = argparse.ArgumentParser()
 182     parser.add_argument(
 183         "--all-json", type=os.path.abspath, help="Path to write json output to"
 184     )
 185     parser.add_argument(
 186         "--untriaged",
 187         type=os.path.abspath,
 188         help="Path to write list of regressions with no associated bug",
 189     )
 190     parser.add_argument(
 191         "--platform",
 192         dest="platforms",
 193         action="append",
 194         choices=list(run_infos.keys()),
 195         help="Configurations to compute fission changes for",
 196     )
 197     commandline.add_logging_group(parser)
 198     return parser
 199
 200
 201 def allowed_results(test, subtest=None):
 202     return test.expected(subtest), test.known_intermittent(subtest)
 203
 204
 205 def is_worse(baseline_result, new_result):
 206     if new_result == baseline_result:
 207         return False
 208
 209     if new_result in ("PASS", "OK"):
 210         return False
 211
 212     if baseline_result in ("PASS", "OK"):
 213         return True
 214
 215     # A crash -> not crash isn't a regression
 216     if baseline_result == "CRASH":
 217         return False
 218
 219     return True
 220
 221
 222 def is_regression(baseline_result, new_result):
 223     if baseline_result == new_result:
 224         return False
 225
 226     baseline_expected, baseline_intermittent = baseline_result
 227     new_expected, new_intermittent = new_result
 228
 229     baseline_all = {baseline_expected} | set(baseline_intermittent)
 230     new_all = {new_expected} | set(new_intermittent)
 231
 232     if baseline_all == new_all:
 233         return False
 234
 235     if not baseline_intermittent and not new_intermittent:
 236         return is_worse(baseline_expected, new_expected)
 237
 238     # If it was intermittent and isn't now, check if the new result is
 239     # worse than any of the previous results so that [PASS, FAIL] -> FAIL
 240     # looks like a regression
 241     if baseline_intermittent and not new_intermittent:
 242         return any(is_worse(result, new_expected) for result in baseline_all)
 243
 244     # If it was a perma and is now intermittent, check if any new result is
 245     # worse than the previous result.
 246     if not baseline_intermittent and new_intermittent:
 247         return any(is_worse(baseline_expected, result) for result in new_all)
 248
 249     # If it was an intermittent and is still an intermittent
 250     # check if any new result not in the old results is worse than
 251     # any old result
 252     new_results = new_all - baseline_all
 253     return any(
 254         is_worse(baseline_result, new_result)
 255         for new_result in new_results
 256         for baseline_result in baseline_all
 257     )
 258
 259
 260 def get_meta_prop(test, subtest, name):
 261     for meta in test.itermeta(subtest):
 262         try:
 263             value = meta.get(name)
 264         except KeyError:
 265             pass
 266         else:
 267             return value
 268     return None
 269
 270
 271 def include_result(result):
 272     if result.disabled or result.regressions:
 273         return True
 274
 275     if isinstance(result, TestResult):
 276         for subtest_result in result.subtest_results.values():
 277             if subtest_result.disabled or subtest_result.regressions:
 278                 return True
 279
 280     return False
 281
 282
 283 class Result:
 284     def __init__(self):
 285         self.bugs = set()
 286         self.disabled = set()
 287         self.regressions = {}
 288
 289     def add_regression(self, platform, baseline_results, fission_results):
 290         self.regressions[platform] = {
 291             "baseline": [baseline_results[0]] + baseline_results[1],
 292             "fission": [fission_results[0]] + fission_results[1],
 293         }
 294
 295     def to_json(self):
 296         raise NotImplementedError
 297
 298     def is_triaged(self):
 299         raise NotImplementedError
 300
 301
 302 class TestResult(Result):
 303     def __init__(self):
 304         super().__init__()
 305         self.subtest_results = {}
 306
 307     def add_subtest(self, name):
 308         self.subtest_results[name] = SubtestResult(self)
 309
 310     def to_json(self):
 311         rv = {}
 312         include_subtests = {
 313             name: item.to_json()
 314             for name, item in self.subtest_results.items()
 315             if include_result(item)
 316         }
 317         if include_subtests:
 318             rv["subtest_results"] = include_subtests
 319         if self.regressions:
 320             rv["regressions"] = self.regressions
 321         if self.disabled:
 322             rv["disabled"] = list(self.disabled)
 323         if self.bugs:
 324             rv["bugs"] = list(self.bugs)
 325         return rv
 326
 327     def is_triaged(self):
 328         return bool(self.bugs) or (
 329             not self.regressions
 330             and all(
 331                 subtest_result.is_triaged()
 332                 for subtest_result in self.subtest_results.values()
 333             )
 334         )
 335
 336
 337 class SubtestResult(Result):
 338     def __init__(self, parent):
 339         super().__init__()
 340         self.parent = parent
 341
 342     def to_json(self):
 343         rv = {}
 344         if self.regressions:
 345             rv["regressions"] = self.regressions
 346         if self.disabled:
 347             rv["disabled"] = list(self.disabled)
 348         bugs = self.bugs - self.parent.bugs
 349         if bugs:
 350             rv["bugs"] = list(bugs)
 351         return rv
 352
 353     def is_triaged(self):
 354         return bool(not self.regressions or self.parent.bugs or self.bugs)
 355
 356
 357 def run(logger, src_root, obj_root, **kwargs):
 358     commandline.setup_logging(
 359         logger, {key: value for key, value in kwargs.items() if key.startswith("log_")}
 360     )
 361
 362     import manifestupdate
 363
 364     sys.path.insert(
 365         0,
 366         os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 367     )
 368     from wptrunner import testloader, wpttest
 369
 370     logger.info("Loading test manifest")
 371     test_manifests = manifestupdate.run(src_root, obj_root, logger)
 372
 373     test_results = {}
 374
 375     platforms = kwargs["platforms"]
 376     if platforms is None:
 377         platforms = run_infos.keys()
 378
 379     for platform in platforms:
 380         platform_run_info = run_infos[platform]
 381         run_info_baseline = platform_run_info.copy()
 382         run_info_baseline["fission"] = False
 383
 384         tests = {}
 385
 386         for kind in ("baseline", "fission"):
 387             logger.info("Loading tests %s %s" % (platform, kind))
 388             run_info = platform_run_info.copy()
 389             run_info["fission"] = kind == "fission"
 390
 391             test_loader = testloader.TestLoader(
 392                 test_manifests, wpttest.enabled_tests, run_info, manifest_filters=[]
 393             )
 394             tests[kind] = {
 395                 test.id: test
 396                 for _, _, test in test_loader.iter_tests()
 397                 if test._test_metadata is not None
 398             }
 399
 400         for test_id, baseline_test in tests["baseline"].items():
 401             fission_test = tests["fission"][test_id]
 402
 403             if test_id not in test_results:
 404                 test_results[test_id] = TestResult()
 405
 406             test_result = test_results[test_id]
 407
 408             baseline_bug = get_meta_prop(baseline_test, None, "bug")
 409             fission_bug = get_meta_prop(fission_test, None, "bug")
 410             if fission_bug and fission_bug != baseline_bug:
 411                 test_result.bugs.add(fission_bug)
 412
 413             if fission_test.disabled() and not baseline_test.disabled():
 414                 test_result.disabled.add(platform)
 415                 reason = get_meta_prop(fission_test, None, "disabled")
 416                 if reason and maybe_bug_re.match(reason):
 417                     test_result.bugs.add(reason)
 418
 419             baseline_results = allowed_results(baseline_test)
 420             fission_results = allowed_results(fission_test)
 421             result_is_regression = is_regression(baseline_results, fission_results)
 422
 423             if baseline_results != fission_results:
 424                 logger.debug(
 425                     "  %s %s %s %s"
 426                     % (test_id, baseline_results, fission_results, result_is_regression)
 427                 )
 428
 429             if result_is_regression:
 430                 test_result.add_regression(platform, baseline_results, fission_results)
 431
 432             for (
 433                 name,
 434                 baseline_subtest_meta,
 435             ) in baseline_test._test_metadata.subtests.items():
 436                 fission_subtest_meta = baseline_test._test_metadata.subtests[name]
 437                 if name not in test_result.subtest_results:
 438                     test_result.add_subtest(name)
 439
 440                 subtest_result = test_result.subtest_results[name]
 441
 442                 baseline_bug = get_meta_prop(baseline_test, name, "bug")
 443                 fission_bug = get_meta_prop(fission_test, name, "bug")
 444                 if fission_bug and fission_bug != baseline_bug:
 445                     subtest_result.bugs.add(fission_bug)
 446
 447                 if bool(fission_subtest_meta.disabled) and not bool(
 448                     baseline_subtest_meta.disabled
 449                 ):
 450                     subtest_result.disabled.add(platform)
 451                     if maybe_bug_re.match(fission_subtest_meta.disabled):
 452                         subtest_result.bugs.add(fission_subtest_meta.disabled)
 453
 454                 baseline_results = allowed_results(baseline_test, name)
 455                 fission_results = allowed_results(fission_test, name)
 456
 457                 result_is_regression = is_regression(baseline_results, fission_results)
 458
 459                 if baseline_results != fission_results:
 460                     logger.debug(
 461                         "    %s %s %s %s %s"
 462                         % (
 463                             test_id,
 464                             name,
 465                             baseline_results,
 466                             fission_results,
 467                             result_is_regression,
 468                         )
 469                     )
 470
 471                 if result_is_regression:
 472                     subtest_result.add_regression(
 473                         platform, baseline_results, fission_results
 474                     )
 475
 476     test_results = {
 477         test_id: result
 478         for test_id, result in test_results.items()
 479         if include_result(result)
 480     }
 481
 482     if kwargs["all_json"] is not None:
 483         write_all(test_results, kwargs["all_json"])
 484
 485     if kwargs["untriaged"] is not None:
 486         write_untriaged(test_results, kwargs["untriaged"])
 487
 488
 489 def write_all(test_results, path):
 490     json_data = {test_id: result.to_json() for test_id, result in test_results.items()}
 491
 492     dir_name = os.path.dirname(path)
 493     if not os.path.exists(dir_name):
 494         os.makedirs(dir_name)
 495
 496     with open(path, "w") as f:
 497         json.dump(json_data, f, indent=2)
 498
 499
 500 def write_untriaged(test_results, path):
 501     dir_name = os.path.dirname(path)
 502     if not os.path.exists(dir_name):
 503         os.makedirs(dir_name)
 504
 505     data = sorted(
 506         (test_id, result)
 507         for test_id, result in test_results.items()
 508         if not result.is_triaged()
 509     )
 510
 511     with open(path, "w") as f:
 512         for test_id, result in data:
 513             f.write(test_id + "\n")
 514             for name, subtest_result in sorted(result.subtest_results.items()):
 515                 if not subtest_result.is_triaged():
 516                     f.write("    %s\n" % name)