testing/web-platform/fissionregressions.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import os
   8 import re
   9 import sys
  10
  11 from mozlog import commandline
  12
  13 run_infos = {
  14     "linux-opt": {
  15         "os": "linux",
  16         "processor": "x86_64",
  17         "version": "Ubuntu 18.04",
  18         "os_version": "18.04",
  19         "bits": 64,
  20         "has_sandbox": True,
  21         "automation": True,
  22         "linux_distro": "Ubuntu",
  23         "apple_silicon": False,
  24         "appname": "firefox",
  25         "artifact": False,
  26         "asan": False,
  27         "bin_suffix": "",
  28         "buildapp": "browser",
  29         "buildtype_guess": "pgo",
  30         "cc_type": "clang",
  31         "ccov": False,
  32         "crashreporter": True,
  33         "datareporting": True,
  34         "debug": False,
  35         "devedition": False,
  36         "early_beta_or_earlier": True,
  37         "healthreport": True,
  38         "nightly_build": True,
  39         "non_native_theme": True,
  40         "normandy": True,
  41         "official": True,
  42         "pgo": True,
  43         "platform_guess": "linux64",
  44         "release_or_beta": False,
  45         "require_signing": False,
  46         "stylo": True,
  47         "sync": True,
  48         "telemetry": False,
  49         "tests_enabled": True,
  50         "toolkit": "gtk",
  51         "tsan": False,
  52         "ubsan": False,
  53         "updater": True,
  54         "python_version": 3,
  55         "product": "firefox",
  56         "verify": False,
  57         "wasm": True,
  58         "e10s": True,
  59         "headless": False,
  60         "fission": True,
  61         "sessionHistoryInParent": True,
  62         "swgl": False,
  63         "win10_2004": False,
  64         "domstreams": False,
  65     },
  66     "linux-debug": {
  67         "os": "linux",
  68         "processor": "x86_64",
  69         "version": "Ubuntu 18.04",
  70         "os_version": "18.04",
  71         "bits": 64,
  72         "has_sandbox": True,
  73         "automation": True,
  74         "linux_distro": "Ubuntu",
  75         "apple_silicon": False,
  76         "appname": "firefox",
  77         "artifact": False,
  78         "asan": False,
  79         "bin_suffix": "",
  80         "buildapp": "browser",
  81         "buildtype_guess": "debug",
  82         "cc_type": "clang",
  83         "ccov": False,
  84         "crashreporter": True,
  85         "datareporting": True,
  86         "debug": True,
  87         "devedition": False,
  88         "early_beta_or_earlier": True,
  89         "healthreport": True,
  90         "nightly_build": True,
  91         "non_native_theme": True,
  92         "normandy": True,
  93         "official": True,
  94         "pgo": False,
  95         "platform_guess": "linux64",
  96         "release_or_beta": False,
  97         "require_signing": False,
  98         "stylo": True,
  99         "sync": True,
 100         "telemetry": False,
 101         "tests_enabled": True,
 102         "toolkit": "gtk",
 103         "tsan": False,
 104         "ubsan": False,
 105         "updater": True,
 106         "python_version": 3,
 107         "product": "firefox",
 108         "verify": False,
 109         "wasm": True,
 110         "e10s": True,
 111         "headless": False,
 112         "fission": False,
 113         "sessionHistoryInParent": False,
 114         "swgl": False,
 115         "win10_2004": False,
 116         "domstreams": False,
 117     },
 118     "win-opt": {
 119         "os": "win",
 120         "processor": "x86_64",
 121         "version": "10.0.17134",
 122         "os_version": "10.0",
 123         "bits": 64,
 124         "has_sandbox": True,
 125         "automation": True,
 126         "service_pack": "",
 127         "apple_silicon": False,
 128         "appname": "firefox",
 129         "artifact": False,
 130         "asan": False,
 131         "bin_suffix": ".exe",
 132         "buildapp": "browser",
 133         "buildtype_guess": "pgo",
 134         "cc_type": "clang-cl",
 135         "ccov": False,
 136         "crashreporter": True,
 137         "datareporting": True,
 138         "debug": False,
 139         "devedition": False,
 140         "early_beta_or_earlier": True,
 141         "healthreport": True,
 142         "nightly_build": True,
 143         "non_native_theme": False,
 144         "normandy": True,
 145         "official": True,
 146         "pgo": True,
 147         "platform_guess": "win64",
 148         "release_or_beta": False,
 149         "require_signing": False,
 150         "stylo": True,
 151         "sync": True,
 152         "telemetry": False,
 153         "tests_enabled": True,
 154         "toolkit": "windows",
 155         "tsan": False,
 156         "ubsan": False,
 157         "updater": True,
 158         "python_version": 3,
 159         "product": "firefox",
 160         "verify": False,
 161         "wasm": True,
 162         "e10s": True,
 163         "headless": False,
 164         "fission": False,
 165         "sessionHistoryInParent": False,
 166         "swgl": False,
 167         "win10_2004": False,
 168         "domstreams": False,
 169     },
 170 }
 171
 172
 173 # RE that checks for anything containing a three+ digit number
 174 maybe_bug_re = re.compile(r".*\d\d\d+")
 175
 176
 177 def get_parser():
 178     parser = argparse.ArgumentParser()
 179     parser.add_argument(
 180         "--all-json", type=os.path.abspath, help="Path to write json output to"
 181     )
 182     parser.add_argument(
 183         "--untriaged",
 184         type=os.path.abspath,
 185         help="Path to write list of regressions with no associated bug",
 186     )
 187     parser.add_argument(
 188         "--platform",
 189         dest="platforms",
 190         action="append",
 191         choices=list(run_infos.keys()),
 192         help="Configurations to compute fission changes for",
 193     )
 194     commandline.add_logging_group(parser)
 195     return parser
 196
 197
 198 def allowed_results(test, subtest=None):
 199     return test.expected(subtest), test.known_intermittent(subtest)
 200
 201
 202 def is_worse(baseline_result, new_result):
 203     if new_result == baseline_result:
 204         return False
 205
 206     if new_result in ("PASS", "OK"):
 207         return False
 208
 209     if baseline_result in ("PASS", "OK"):
 210         return True
 211
 212     # A crash -> not crash isn't a regression
 213     if baseline_result == "CRASH":
 214         return False
 215
 216     return True
 217
 218
 219 def is_regression(baseline_result, new_result):
 220     if baseline_result == new_result:
 221         return False
 222
 223     baseline_expected, baseline_intermittent = baseline_result
 224     new_expected, new_intermittent = new_result
 225
 226     baseline_all = {baseline_expected} | set(baseline_intermittent)
 227     new_all = {new_expected} | set(new_intermittent)
 228
 229     if baseline_all == new_all:
 230         return False
 231
 232     if not baseline_intermittent and not new_intermittent:
 233         return is_worse(baseline_expected, new_expected)
 234
 235     # If it was intermittent and isn't now, check if the new result is
 236     # worse than any of the previous results so that [PASS, FAIL] -> FAIL
 237     # looks like a regression
 238     if baseline_intermittent and not new_intermittent:
 239         return any(is_worse(result, new_expected) for result in baseline_all)
 240
 241     # If it was a perma and is now intermittent, check if any new result is
 242     # worse than the previous result.
 243     if not baseline_intermittent and new_intermittent:
 244         return any(is_worse(baseline_expected, result) for result in new_all)
 245
 246     # If it was an intermittent and is still an intermittent
 247     # check if any new result not in the old results is worse than
 248     # any old result
 249     new_results = new_all - baseline_all
 250     return any(
 251         is_worse(baseline_result, new_result)
 252         for new_result in new_results
 253         for baseline_result in baseline_all
 254     )
 255
 256
 257 def get_meta_prop(test, subtest, name):
 258     for meta in test.itermeta(subtest):
 259         try:
 260             value = meta.get(name)
 261         except KeyError:
 262             pass
 263         else:
 264             return value
 265     return None
 266
 267
 268 def include_result(result):
 269     if result.disabled or result.regressions:
 270         return True
 271
 272     if isinstance(result, TestResult):
 273         for subtest_result in result.subtest_results.values():
 274             if subtest_result.disabled or subtest_result.regressions:
 275                 return True
 276
 277     return False
 278
 279
 280 class Result:
 281     def __init__(self):
 282         self.bugs = set()
 283         self.disabled = set()
 284         self.regressions = {}
 285
 286     def add_regression(self, platform, baseline_results, fission_results):
 287         self.regressions[platform] = {
 288             "baseline": [baseline_results[0]] + baseline_results[1],
 289             "fission": [fission_results[0]] + fission_results[1],
 290         }
 291
 292     def to_json(self):
 293         raise NotImplementedError
 294
 295     def is_triaged(self):
 296         raise NotImplementedError
 297
 298
 299 class TestResult(Result):
 300     def __init__(self):
 301         super().__init__()
 302         self.subtest_results = {}
 303
 304     def add_subtest(self, name):
 305         self.subtest_results[name] = SubtestResult(self)
 306
 307     def to_json(self):
 308         rv = {}
 309         include_subtests = {
 310             name: item.to_json()
 311             for name, item in self.subtest_results.items()
 312             if include_result(item)
 313         }
 314         if include_subtests:
 315             rv["subtest_results"] = include_subtests
 316         if self.regressions:
 317             rv["regressions"] = self.regressions
 318         if self.disabled:
 319             rv["disabled"] = list(self.disabled)
 320         if self.bugs:
 321             rv["bugs"] = list(self.bugs)
 322         return rv
 323
 324     def is_triaged(self):
 325         return bool(self.bugs) or (
 326             not self.regressions
 327             and all(
 328                 subtest_result.is_triaged()
 329                 for subtest_result in self.subtest_results.values()
 330             )
 331         )
 332
 333
 334 class SubtestResult(Result):
 335     def __init__(self, parent):
 336         super().__init__()
 337         self.parent = parent
 338
 339     def to_json(self):
 340         rv = {}
 341         if self.regressions:
 342             rv["regressions"] = self.regressions
 343         if self.disabled:
 344             rv["disabled"] = list(self.disabled)
 345         bugs = self.bugs - self.parent.bugs
 346         if bugs:
 347             rv["bugs"] = list(bugs)
 348         return rv
 349
 350     def is_triaged(self):
 351         return bool(not self.regressions or self.parent.bugs or self.bugs)
 352
 353
 354 def run(logger, src_root, obj_root, **kwargs):
 355     commandline.setup_logging(
 356         logger, {key: value for key, value in kwargs.items() if key.startswith("log_")}
 357     )
 358
 359     import manifestupdate
 360
 361     sys.path.insert(
 362         0,
 363         os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 364     )
 365     from wptrunner import testloader, wpttest
 366
 367     logger.info("Loading test manifest")
 368     test_manifests = manifestupdate.run(src_root, obj_root, logger)
 369
 370     test_results = {}
 371
 372     platforms = kwargs["platforms"]
 373     if platforms is None:
 374         platforms = run_infos.keys()
 375
 376     for platform in platforms:
 377         platform_run_info = run_infos[platform]
 378         run_info_baseline = platform_run_info.copy()
 379         run_info_baseline["fission"] = False
 380
 381         tests = {}
 382
 383         for kind in ("baseline", "fission"):
 384             logger.info("Loading tests %s %s" % (platform, kind))
 385             run_info = platform_run_info.copy()
 386             run_info["fission"] = kind == "fission"
 387
 388             test_loader = testloader.TestLoader(
 389                 test_manifests, wpttest.enabled_tests, run_info, manifest_filters=[]
 390             )
 391             tests[kind] = {
 392                 test.id: test
 393                 for _, _, test in test_loader.iter_tests()
 394                 if test._test_metadata is not None
 395             }
 396
 397         for test_id, baseline_test in tests["baseline"].items():
 398             fission_test = tests["fission"][test_id]
 399
 400             if test_id not in test_results:
 401                 test_results[test_id] = TestResult()
 402
 403             test_result = test_results[test_id]
 404
 405             baseline_bug = get_meta_prop(baseline_test, None, "bug")
 406             fission_bug = get_meta_prop(fission_test, None, "bug")
 407             if fission_bug and fission_bug != baseline_bug:
 408                 test_result.bugs.add(fission_bug)
 409
 410             if fission_test.disabled() and not baseline_test.disabled():
 411                 test_result.disabled.add(platform)
 412                 reason = get_meta_prop(fission_test, None, "disabled")
 413                 if reason and maybe_bug_re.match(reason):
 414                     test_result.bugs.add(reason)
 415
 416             baseline_results = allowed_results(baseline_test)
 417             fission_results = allowed_results(fission_test)
 418             result_is_regression = is_regression(baseline_results, fission_results)
 419
 420             if baseline_results != fission_results:
 421                 logger.debug(
 422                     "  %s %s %s %s"
 423                     % (test_id, baseline_results, fission_results, result_is_regression)
 424                 )
 425
 426             if result_is_regression:
 427                 test_result.add_regression(platform, baseline_results, fission_results)
 428
 429             for (
 430                 name,
 431                 baseline_subtest_meta,
 432             ) in baseline_test._test_metadata.subtests.items():
 433                 fission_subtest_meta = baseline_test._test_metadata.subtests[name]
 434                 if name not in test_result.subtest_results:
 435                     test_result.add_subtest(name)
 436
 437                 subtest_result = test_result.subtest_results[name]
 438
 439                 baseline_bug = get_meta_prop(baseline_test, name, "bug")
 440                 fission_bug = get_meta_prop(fission_test, name, "bug")
 441                 if fission_bug and fission_bug != baseline_bug:
 442                     subtest_result.bugs.add(fission_bug)
 443
 444                 if bool(fission_subtest_meta.disabled) and not bool(
 445                     baseline_subtest_meta.disabled
 446                 ):
 447                     subtest_result.disabled.add(platform)
 448                     if maybe_bug_re.match(fission_subtest_meta.disabled):
 449                         subtest_result.bugs.add(fission_subtest_meta.disabled)
 450
 451                 baseline_results = allowed_results(baseline_test, name)
 452                 fission_results = allowed_results(fission_test, name)
 453
 454                 result_is_regression = is_regression(baseline_results, fission_results)
 455
 456                 if baseline_results != fission_results:
 457                     logger.debug(
 458                         "    %s %s %s %s %s"
 459                         % (
 460                             test_id,
 461                             name,
 462                             baseline_results,
 463                             fission_results,
 464                             result_is_regression,
 465                         )
 466                     )
 467
 468                 if result_is_regression:
 469                     subtest_result.add_regression(
 470                         platform, baseline_results, fission_results
 471                     )
 472
 473     test_results = {
 474         test_id: result
 475         for test_id, result in test_results.items()
 476         if include_result(result)
 477     }
 478
 479     if kwargs["all_json"] is not None:
 480         write_all(test_results, kwargs["all_json"])
 481
 482     if kwargs["untriaged"] is not None:
 483         write_untriaged(test_results, kwargs["untriaged"])
 484
 485
 486 def write_all(test_results, path):
 487     json_data = {test_id: result.to_json() for test_id, result in test_results.items()}
 488
 489     dir_name = os.path.dirname(path)
 490     if not os.path.exists(dir_name):
 491         os.makedirs(dir_name)
 492
 493     with open(path, "w") as f:
 494         json.dump(json_data, f, indent=2)
 495
 496
 497 def write_untriaged(test_results, path):
 498     dir_name = os.path.dirname(path)
 499     if not os.path.exists(dir_name):
 500         os.makedirs(dir_name)
 501
 502     data = sorted(
 503         (test_id, result)
 504         for test_id, result in test_results.items()
 505         if not result.is_triaged()
 506     )
 507
 508     with open(path, "w") as f:
 509         for test_id, result in data:
 510             f.write(test_id + "\n")
 511             for name, subtest_result in sorted(result.subtest_results.items()):
 512                 if not subtest_result.is_triaged():
 513                     f.write("    %s\n" % name)