8 "Class for creating, bisecting and summarizing for --bisect-chunk option."
10 def __init__(self
, harness
):
11 super(Bisect
, self
).__init
__()
18 def setup(self
, tests
):
19 """This method is used to initialize various variables that are required
23 # We need totalTests key in contents for sanity check
24 self
.contents
["totalTests"] = tests
25 self
.contents
["tests"] = tests
26 self
.contents
["loop"] = 0
29 def reset(self
, expectedError
, result
):
30 """This method is used to initialize self.expectedError and self.result
31 for each loop in runtests."""
32 self
.expectedError
= expectedError
35 def get_tests_for_bisection(self
, options
, tests
):
36 """Make a list of tests for bisection from a given list of tests"""
39 bisectlist
.append(test
)
40 if test
.endswith(options
.bisectChunk
):
45 def pre_test(self
, options
, tests
, status
):
46 """This method is used to call other methods for setting up variables and
47 getting the list of tests for bisection."""
48 if options
.bisectChunk
== "default":
50 # The second condition in 'if' is required to verify that the failing
51 # test is the last one.
52 elif "loop" not in self
.contents
or not self
.contents
["tests"][-1].endswith(
55 tests
= self
.get_tests_for_bisection(options
, tests
)
56 status
= self
.setup(tests
)
58 return self
.next_chunk_binary(options
, status
)
60 def post_test(self
, options
, expectedError
, result
):
61 """This method is used to call other methods to summarize results and check whether a
62 sanity check is done or not."""
63 self
.reset(expectedError
, result
)
64 status
= self
.summarize_chunk(options
)
65 # Check whether sanity check has to be done. Also it is necessary to check whether
66 # options.bisectChunk is present in self.expectedError as we do not want to run
68 if status
== -1 and options
.bisectChunk
in self
.expectedError
:
69 # In case we have a debug build, we don't want to run a sanity
70 # check, will take too much time.
71 if mozinfo
.info
["debug"]:
74 testBleedThrough
= self
.contents
["testsToRun"][0]
75 tests
= self
.contents
["totalTests"]
76 tests
.remove(testBleedThrough
)
77 # To make sure that the failing test is dependent on some other
79 if options
.bisectChunk
in testBleedThrough
:
82 status
= self
.setup(tests
)
83 self
.summary
.append("Sanity Check:")
87 def next_chunk_reverse(self
, options
, status
):
88 "This method is used to bisect the tests in a reverse search fashion."
91 if self
.contents
["loop"] <= 1:
92 self
.contents
["testsToRun"] = self
.contents
["tests"]
93 if self
.contents
["loop"] == 1:
94 self
.contents
["testsToRun"] = [self
.contents
["tests"][-1]]
95 self
.contents
["loop"] += 1
96 return self
.contents
["testsToRun"]
98 if "result" in self
.contents
:
99 if self
.contents
["result"] == "PASS":
100 chunkSize
= self
.contents
["end"] - self
.contents
["start"]
101 self
.contents
["end"] = self
.contents
["start"] - 1
102 self
.contents
["start"] = self
.contents
["end"] - chunkSize
104 # self.contents['result'] will be expected error only if it fails.
105 elif self
.contents
["result"] == "FAIL":
106 self
.contents
["tests"] = self
.contents
["testsToRun"]
107 status
= 1 # for initializing
111 totalTests
= len(self
.contents
["tests"])
112 chunkSize
= int(math
.ceil(totalTests
/ 10.0))
113 self
.contents
["start"] = totalTests
- chunkSize
- 1
114 self
.contents
["end"] = totalTests
- 2
116 start
= self
.contents
["start"]
117 end
= self
.contents
["end"] + 1
118 self
.contents
["testsToRun"] = self
.contents
["tests"][start
:end
]
119 self
.contents
["testsToRun"].append(self
.contents
["tests"][-1])
120 self
.contents
["loop"] += 1
122 return self
.contents
["testsToRun"]
124 def next_chunk_binary(self
, options
, status
):
125 "This method is used to bisect the tests in a binary search fashion."
128 if self
.contents
["loop"] <= 1:
129 self
.contents
["testsToRun"] = self
.contents
["tests"]
130 if self
.contents
["loop"] == 1:
131 self
.contents
["testsToRun"] = [self
.contents
["tests"][-1]]
132 self
.contents
["loop"] += 1
133 return self
.contents
["testsToRun"]
135 # Initialize the contents dict.
137 totalTests
= len(self
.contents
["tests"])
138 self
.contents
["start"] = 0
139 self
.contents
["end"] = totalTests
- 2
141 # pylint --py3k W1619
142 mid
= (self
.contents
["start"] + self
.contents
["end"]) / 2
143 if "result" in self
.contents
:
144 if self
.contents
["result"] == "PASS":
145 self
.contents
["end"] = mid
147 elif self
.contents
["result"] == "FAIL":
148 self
.contents
["start"] = mid
+ 1
150 mid
= (self
.contents
["start"] + self
.contents
["end"]) / 2
152 end
= self
.contents
["end"] + 1
153 self
.contents
["testsToRun"] = self
.contents
["tests"][start
:end
]
154 if not self
.contents
["testsToRun"]:
155 self
.contents
["testsToRun"].append(self
.contents
["tests"][mid
])
156 self
.contents
["testsToRun"].append(self
.contents
["tests"][-1])
157 self
.contents
["loop"] += 1
159 return self
.contents
["testsToRun"]
161 def summarize_chunk(self
, options
):
162 "This method is used summarize the results after the list of tests is run."
163 if options
.bisectChunk
== "default":
164 # if no expectedError that means all the tests have successfully
166 if len(self
.expectedError
) == 0:
168 options
.bisectChunk
= self
.expectedError
.keys()[0]
169 self
.summary
.append("\tFound Error in test: %s" % options
.bisectChunk
)
172 # If options.bisectChunk is not in self.result then we need to move to
174 if options
.bisectChunk
not in self
.result
:
177 self
.summary
.append("\tPass %d:" % self
.contents
["loop"])
178 if len(self
.contents
["testsToRun"]) > 1:
180 "\t\t%d test files(start,end,failing). [%s, %s, %s]"
182 len(self
.contents
["testsToRun"]),
183 self
.contents
["testsToRun"][0],
184 self
.contents
["testsToRun"][-2],
185 self
.contents
["testsToRun"][-1],
189 self
.summary
.append("\t\t1 test file [%s]" % self
.contents
["testsToRun"][0])
190 return self
.check_for_intermittent(options
)
192 if self
.result
[options
.bisectChunk
] == "PASS":
193 self
.summary
.append("\t\tno failures found.")
194 if self
.contents
["loop"] == 1:
197 self
.contents
["result"] = "PASS"
200 elif self
.result
[options
.bisectChunk
] == "FAIL":
201 if "expectedError" not in self
.contents
:
202 self
.summary
.append("\t\t%s failed." % self
.contents
["testsToRun"][-1])
203 self
.contents
["expectedError"] = self
.expectedError
[options
.bisectChunk
]
207 self
.expectedError
[options
.bisectChunk
]
208 == self
.contents
["expectedError"]
211 "\t\t%s failed with expected error."
212 % self
.contents
["testsToRun"][-1]
214 self
.contents
["result"] = "FAIL"
217 # This code checks for test-bleedthrough. Should work for any
219 numberOfTests
= len(self
.contents
["testsToRun"])
220 if numberOfTests
< 3:
221 # This means that only 2 tests are run. Since the last test
222 # is the failing test itself therefore the bleedthrough
223 # test is the first test
225 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
226 "root cause for many of the above failures"
227 % self
.contents
["testsToRun"][0]
232 "\t\t%s failed with different error."
233 % self
.contents
["testsToRun"][-1]
239 def check_for_intermittent(self
, options
):
240 "This method is used to check whether a test is an intermittent."
241 if self
.result
[options
.bisectChunk
] == "PASS":
243 "\t\tThe test %s passed." % self
.contents
["testsToRun"][0]
246 # loop is set to 1 to again run the single test.
247 self
.contents
["loop"] = 1
251 if self
.failcount
> 0:
252 # -1 is being returned as the test is intermittent, so no need to bisect
255 # If the test does not fail even once, then proceed to next chunk for bisection.
256 # loop is set to 2 to proceed on bisection.
257 self
.contents
["loop"] = 2
259 elif self
.result
[options
.bisectChunk
] == "FAIL":
261 "\t\tThe test %s failed." % self
.contents
["testsToRun"][0]
264 self
.contents
["loop"] = 1
266 # self.max_failures is the maximum number of times a test is allowed
267 # to fail to be called an intermittent. If a test fails more than
268 # limit set, it is a perma-fail.
269 if self
.failcount
< self
.max_failures
:
271 # -1 is being returned as the test is intermittent, so no need to bisect
277 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
278 "root cause for many of the above failures"
279 % self
.contents
["testsToRun"][0]
283 def print_summary(self
):
284 "This method is used to print the recorded summary."
285 print("Bisection summary:")
286 for line
in self
.summary
: