1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
11 from mozlog
import commandline
16 "processor": "x86_64",
17 "version": "Ubuntu 18.04",
18 "os_version": "18.04",
22 "linux_distro": "Ubuntu",
23 "apple_silicon": False,
28 "buildapp": "browser",
29 "buildtype_guess": "pgo",
32 "crashreporter": True,
33 "datareporting": True,
36 "early_beta_or_earlier": True,
38 "nightly_build": True,
42 "platform_guess": "linux64",
43 "release_or_beta": False,
44 "require_signing": False,
48 "tests_enabled": True,
60 "sessionHistoryInParent": True,
62 "privateBrowsing": False,
65 "isolated_process": False,
70 "processor": "x86_64",
71 "version": "Ubuntu 18.04",
72 "os_version": "18.04",
76 "linux_distro": "Ubuntu",
77 "apple_silicon": False,
82 "buildapp": "browser",
83 "buildtype_guess": "debug",
86 "crashreporter": True,
87 "datareporting": True,
90 "early_beta_or_earlier": True,
92 "nightly_build": True,
96 "platform_guess": "linux64",
97 "release_or_beta": False,
98 "require_signing": False,
102 "tests_enabled": True,
108 "product": "firefox",
114 "sessionHistoryInParent": False,
116 "privateBrowsing": False,
119 "isolated_process": False,
124 "processor": "x86_64",
125 "version": "10.0.17134",
126 "os_version": "10.0",
130 "apple_silicon": False,
131 "appname": "firefox",
134 "bin_suffix": ".exe",
135 "buildapp": "browser",
136 "buildtype_guess": "pgo",
137 "cc_type": "clang-cl",
139 "crashreporter": True,
140 "datareporting": True,
143 "early_beta_or_earlier": True,
144 "healthreport": True,
145 "nightly_build": True,
149 "platform_guess": "win64",
150 "release_or_beta": False,
151 "require_signing": False,
155 "tests_enabled": True,
156 "toolkit": "windows",
161 "product": "firefox",
167 "sessionHistoryInParent": False,
169 "privateBrowsing": False,
172 "isolated_process": False,
178 # RE that checks for anything containing a three+ digit number
179 maybe_bug_re
= re
.compile(r
".*\d\d\d+")
183 parser
= argparse
.ArgumentParser()
185 "--all-json", type=os
.path
.abspath
, help="Path to write json output to"
189 type=os
.path
.abspath
,
190 help="Path to write list of regressions with no associated bug",
196 choices
=list(run_infos
.keys()),
197 help="Configurations to compute fission changes for",
199 commandline
.add_logging_group(parser
)
203 def allowed_results(test
, subtest
=None):
204 return test
.expected(subtest
), test
.known_intermittent(subtest
)
207 def is_worse(baseline_result
, new_result
):
208 if new_result
== baseline_result
:
211 if new_result
in ("PASS", "OK"):
214 if baseline_result
in ("PASS", "OK"):
217 # A crash -> not crash isn't a regression
218 if baseline_result
== "CRASH":
224 def is_regression(baseline_result
, new_result
):
225 if baseline_result
== new_result
:
228 baseline_expected
, baseline_intermittent
= baseline_result
229 new_expected
, new_intermittent
= new_result
231 baseline_all
= {baseline_expected} |
set(baseline_intermittent
)
232 new_all
= {new_expected} |
set(new_intermittent
)
234 if baseline_all
== new_all
:
237 if not baseline_intermittent
and not new_intermittent
:
238 return is_worse(baseline_expected
, new_expected
)
240 # If it was intermittent and isn't now, check if the new result is
241 # worse than any of the previous results so that [PASS, FAIL] -> FAIL
242 # looks like a regression
243 if baseline_intermittent
and not new_intermittent
:
244 return any(is_worse(result
, new_expected
) for result
in baseline_all
)
246 # If it was a perma and is now intermittent, check if any new result is
247 # worse than the previous result.
248 if not baseline_intermittent
and new_intermittent
:
249 return any(is_worse(baseline_expected
, result
) for result
in new_all
)
251 # If it was an intermittent and is still an intermittent
252 # check if any new result not in the old results is worse than
254 new_results
= new_all
- baseline_all
256 is_worse(baseline_result
, new_result
)
257 for new_result
in new_results
258 for baseline_result
in baseline_all
262 def get_meta_prop(test
, subtest
, name
):
263 for meta
in test
.itermeta(subtest
):
265 value
= meta
.get(name
)
273 def include_result(result
):
274 if result
.disabled
or result
.regressions
:
277 if isinstance(result
, TestResult
):
278 for subtest_result
in result
.subtest_results
.values():
279 if subtest_result
.disabled
or subtest_result
.regressions
:
288 self
.disabled
= set()
289 self
.regressions
= {}
291 def add_regression(self
, platform
, baseline_results
, fission_results
):
292 self
.regressions
[platform
] = {
293 "baseline": [baseline_results
[0]] + baseline_results
[1],
294 "fission": [fission_results
[0]] + fission_results
[1],
298 raise NotImplementedError
300 def is_triaged(self
):
301 raise NotImplementedError
304 class TestResult(Result
):
307 self
.subtest_results
= {}
309 def add_subtest(self
, name
):
310 self
.subtest_results
[name
] = SubtestResult(self
)
316 for name
, item
in self
.subtest_results
.items()
317 if include_result(item
)
320 rv
["subtest_results"] = include_subtests
322 rv
["regressions"] = self
.regressions
324 rv
["disabled"] = list(self
.disabled
)
326 rv
["bugs"] = list(self
.bugs
)
329 def is_triaged(self
):
330 return bool(self
.bugs
) or (
333 subtest_result
.is_triaged()
334 for subtest_result
in self
.subtest_results
.values()
339 class SubtestResult(Result
):
340 def __init__(self
, parent
):
347 rv
["regressions"] = self
.regressions
349 rv
["disabled"] = list(self
.disabled
)
350 bugs
= self
.bugs
- self
.parent
.bugs
352 rv
["bugs"] = list(bugs
)
355 def is_triaged(self
):
356 return bool(not self
.regressions
or self
.parent
.bugs
or self
.bugs
)
359 def run(logger
, src_root
, obj_root
, **kwargs
):
360 commandline
.setup_logging(
361 logger
, {key
: value
for key
, value
in kwargs
.items() if key
.startswith("log_")}
364 import manifestupdate
368 os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), "tests", "tools")),
370 from wptrunner
import testloader
, wpttest
372 logger
.info("Loading test manifest")
373 test_manifests
= manifestupdate
.run(src_root
, obj_root
, logger
)
377 platforms
= kwargs
["platforms"]
378 if platforms
is None:
379 platforms
= run_infos
.keys()
381 for platform
in platforms
:
382 platform_run_info
= run_infos
[platform
]
383 run_info_baseline
= platform_run_info
.copy()
384 run_info_baseline
["fission"] = False
388 for kind
in ("baseline", "fission"):
389 logger
.info("Loading tests %s %s" % (platform
, kind
))
390 run_info
= platform_run_info
.copy()
391 run_info
["fission"] = kind
== "fission"
393 subsuites
= testloader
.load_subsuites(logger
, run_info
, None, set())
394 test_loader
= testloader
.TestLoader(
396 wpttest
.enabled_tests
,
403 for _
, _
, test
in test_loader
.iter_tests(
404 run_info
, test_loader
.manifest_filters
406 if test
._test
_metadata
is not None
409 for test_id
, baseline_test
in tests
["baseline"].items():
410 fission_test
= tests
["fission"][test_id
]
412 if test_id
not in test_results
:
413 test_results
[test_id
] = TestResult()
415 test_result
= test_results
[test_id
]
417 baseline_bug
= get_meta_prop(baseline_test
, None, "bug")
418 fission_bug
= get_meta_prop(fission_test
, None, "bug")
419 if fission_bug
and fission_bug
!= baseline_bug
:
420 test_result
.bugs
.add(fission_bug
)
422 if fission_test
.disabled() and not baseline_test
.disabled():
423 test_result
.disabled
.add(platform
)
424 reason
= get_meta_prop(fission_test
, None, "disabled")
425 if reason
and maybe_bug_re
.match(reason
):
426 test_result
.bugs
.add(reason
)
428 baseline_results
= allowed_results(baseline_test
)
429 fission_results
= allowed_results(fission_test
)
430 result_is_regression
= is_regression(baseline_results
, fission_results
)
432 if baseline_results
!= fission_results
:
435 % (test_id
, baseline_results
, fission_results
, result_is_regression
)
438 if result_is_regression
:
439 test_result
.add_regression(platform
, baseline_results
, fission_results
)
443 baseline_subtest_meta
,
444 ) in baseline_test
._test
_metadata
.subtests
.items():
445 fission_subtest_meta
= baseline_test
._test
_metadata
.subtests
[name
]
446 if name
not in test_result
.subtest_results
:
447 test_result
.add_subtest(name
)
449 subtest_result
= test_result
.subtest_results
[name
]
451 baseline_bug
= get_meta_prop(baseline_test
, name
, "bug")
452 fission_bug
= get_meta_prop(fission_test
, name
, "bug")
453 if fission_bug
and fission_bug
!= baseline_bug
:
454 subtest_result
.bugs
.add(fission_bug
)
456 if bool(fission_subtest_meta
.disabled
) and not bool(
457 baseline_subtest_meta
.disabled
459 subtest_result
.disabled
.add(platform
)
460 if maybe_bug_re
.match(fission_subtest_meta
.disabled
):
461 subtest_result
.bugs
.add(fission_subtest_meta
.disabled
)
463 baseline_results
= allowed_results(baseline_test
, name
)
464 fission_results
= allowed_results(fission_test
, name
)
466 result_is_regression
= is_regression(baseline_results
, fission_results
)
468 if baseline_results
!= fission_results
:
476 result_is_regression
,
480 if result_is_regression
:
481 subtest_result
.add_regression(
482 platform
, baseline_results
, fission_results
487 for test_id
, result
in test_results
.items()
488 if include_result(result
)
491 if kwargs
["all_json"] is not None:
492 write_all(test_results
, kwargs
["all_json"])
494 if kwargs
["untriaged"] is not None:
495 write_untriaged(test_results
, kwargs
["untriaged"])
498 def write_all(test_results
, path
):
499 json_data
= {test_id
: result
.to_json() for test_id
, result
in test_results
.items()}
501 dir_name
= os
.path
.dirname(path
)
502 if not os
.path
.exists(dir_name
):
503 os
.makedirs(dir_name
)
505 with
open(path
, "w") as f
:
506 json
.dump(json_data
, f
, indent
=2)
509 def write_untriaged(test_results
, path
):
510 dir_name
= os
.path
.dirname(path
)
511 if not os
.path
.exists(dir_name
):
512 os
.makedirs(dir_name
)
516 for test_id
, result
in test_results
.items()
517 if not result
.is_triaged()
520 with
open(path
, "w") as f
:
521 for test_id
, result
in data
:
522 f
.write(test_id
+ "\n")
523 for name
, subtest_result
in sorted(result
.subtest_results
.items()):
524 if not subtest_result
.is_triaged():
525 f
.write(" %s\n" % name
)