7 "Class for creating, bisecting and summarizing for --bisect-chunk option."
9 def __init__(self
, harness
):
10 super(Bisect
, self
).__init
__()
17 def setup(self
, tests
):
18 """This method is used to initialize various variables that are required
22 # We need totalTests key in contents for sanity check
23 self
.contents
["totalTests"] = tests
24 self
.contents
["tests"] = tests
25 self
.contents
["loop"] = 0
28 def reset(self
, expectedError
, result
):
29 """This method is used to initialize self.expectedError and self.result
30 for each loop in runtests."""
31 self
.expectedError
= expectedError
34 def get_tests_for_bisection(self
, options
, tests
):
35 """Make a list of tests for bisection from a given list of tests"""
38 bisectlist
.append(test
)
39 if test
.endswith(options
.bisectChunk
):
44 def pre_test(self
, options
, tests
, status
):
45 """This method is used to call other methods for setting up variables and
46 getting the list of tests for bisection."""
47 if options
.bisectChunk
== "default":
49 # The second condition in 'if' is required to verify that the failing
50 # test is the last one.
51 elif "loop" not in self
.contents
or not self
.contents
["tests"][-1].endswith(
54 tests
= self
.get_tests_for_bisection(options
, tests
)
55 status
= self
.setup(tests
)
57 return self
.next_chunk_binary(options
, status
)
59 def post_test(self
, options
, expectedError
, result
):
60 """This method is used to call other methods to summarize results and check whether a
61 sanity check is done or not."""
62 self
.reset(expectedError
, result
)
63 status
= self
.summarize_chunk(options
)
64 # Check whether sanity check has to be done. Also it is necessary to check whether
65 # options.bisectChunk is present in self.expectedError as we do not want to run
67 if status
== -1 and options
.bisectChunk
in self
.expectedError
:
68 # In case we have a debug build, we don't want to run a sanity
69 # check, will take too much time.
70 if mozinfo
.info
["debug"]:
73 testBleedThrough
= self
.contents
["testsToRun"][0]
74 tests
= self
.contents
["totalTests"]
75 tests
.remove(testBleedThrough
)
76 # To make sure that the failing test is dependent on some other
78 if options
.bisectChunk
in testBleedThrough
:
81 status
= self
.setup(tests
)
82 self
.summary
.append("Sanity Check:")
86 def next_chunk_reverse(self
, options
, status
):
87 "This method is used to bisect the tests in a reverse search fashion."
90 if self
.contents
["loop"] <= 1:
91 self
.contents
["testsToRun"] = self
.contents
["tests"]
92 if self
.contents
["loop"] == 1:
93 self
.contents
["testsToRun"] = [self
.contents
["tests"][-1]]
94 self
.contents
["loop"] += 1
95 return self
.contents
["testsToRun"]
97 if "result" in self
.contents
:
98 if self
.contents
["result"] == "PASS":
99 chunkSize
= self
.contents
["end"] - self
.contents
["start"]
100 self
.contents
["end"] = self
.contents
["start"] - 1
101 self
.contents
["start"] = self
.contents
["end"] - chunkSize
103 # self.contents['result'] will be expected error only if it fails.
104 elif self
.contents
["result"] == "FAIL":
105 self
.contents
["tests"] = self
.contents
["testsToRun"]
106 status
= 1 # for initializing
110 totalTests
= len(self
.contents
["tests"])
111 chunkSize
= int(math
.ceil(totalTests
/ 10.0))
112 self
.contents
["start"] = totalTests
- chunkSize
- 1
113 self
.contents
["end"] = totalTests
- 2
115 start
= self
.contents
["start"]
116 end
= self
.contents
["end"] + 1
117 self
.contents
["testsToRun"] = self
.contents
["tests"][start
:end
]
118 self
.contents
["testsToRun"].append(self
.contents
["tests"][-1])
119 self
.contents
["loop"] += 1
121 return self
.contents
["testsToRun"]
123 def next_chunk_binary(self
, options
, status
):
124 "This method is used to bisect the tests in a binary search fashion."
127 if self
.contents
["loop"] <= 1:
128 self
.contents
["testsToRun"] = self
.contents
["tests"]
129 if self
.contents
["loop"] == 1:
130 self
.contents
["testsToRun"] = [self
.contents
["tests"][-1]]
131 self
.contents
["loop"] += 1
132 return self
.contents
["testsToRun"]
134 # Initialize the contents dict.
136 totalTests
= len(self
.contents
["tests"])
137 self
.contents
["start"] = 0
138 self
.contents
["end"] = totalTests
- 2
140 # pylint --py3k W1619
141 mid
= (self
.contents
["start"] + self
.contents
["end"]) / 2
142 if "result" in self
.contents
:
143 if self
.contents
["result"] == "PASS":
144 self
.contents
["end"] = mid
146 elif self
.contents
["result"] == "FAIL":
147 self
.contents
["start"] = mid
+ 1
149 mid
= (self
.contents
["start"] + self
.contents
["end"]) / 2
151 end
= self
.contents
["end"] + 1
152 self
.contents
["testsToRun"] = self
.contents
["tests"][start
:end
]
153 if not self
.contents
["testsToRun"]:
154 self
.contents
["testsToRun"].append(self
.contents
["tests"][mid
])
155 self
.contents
["testsToRun"].append(self
.contents
["tests"][-1])
156 self
.contents
["loop"] += 1
158 return self
.contents
["testsToRun"]
160 def summarize_chunk(self
, options
):
161 "This method is used summarize the results after the list of tests is run."
162 if options
.bisectChunk
== "default":
163 # if no expectedError that means all the tests have successfully
165 if len(self
.expectedError
) == 0:
167 options
.bisectChunk
= self
.expectedError
.keys()[0]
168 self
.summary
.append("\tFound Error in test: %s" % options
.bisectChunk
)
171 # If options.bisectChunk is not in self.result then we need to move to
173 if options
.bisectChunk
not in self
.result
:
176 self
.summary
.append("\tPass %d:" % self
.contents
["loop"])
177 if len(self
.contents
["testsToRun"]) > 1:
179 "\t\t%d test files(start,end,failing). [%s, %s, %s]"
181 len(self
.contents
["testsToRun"]),
182 self
.contents
["testsToRun"][0],
183 self
.contents
["testsToRun"][-2],
184 self
.contents
["testsToRun"][-1],
188 self
.summary
.append("\t\t1 test file [%s]" % self
.contents
["testsToRun"][0])
189 return self
.check_for_intermittent(options
)
191 if self
.result
[options
.bisectChunk
] == "PASS":
192 self
.summary
.append("\t\tno failures found.")
193 if self
.contents
["loop"] == 1:
196 self
.contents
["result"] = "PASS"
199 elif self
.result
[options
.bisectChunk
] == "FAIL":
200 if "expectedError" not in self
.contents
:
201 self
.summary
.append("\t\t%s failed." % self
.contents
["testsToRun"][-1])
202 self
.contents
["expectedError"] = self
.expectedError
[options
.bisectChunk
]
206 self
.expectedError
[options
.bisectChunk
]
207 == self
.contents
["expectedError"]
210 "\t\t%s failed with expected error."
211 % self
.contents
["testsToRun"][-1]
213 self
.contents
["result"] = "FAIL"
216 # This code checks for test-bleedthrough. Should work for any
218 numberOfTests
= len(self
.contents
["testsToRun"])
219 if numberOfTests
< 3:
220 # This means that only 2 tests are run. Since the last test
221 # is the failing test itself therefore the bleedthrough
222 # test is the first test
224 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
225 "root cause for many of the above failures"
226 % self
.contents
["testsToRun"][0]
231 "\t\t%s failed with different error."
232 % self
.contents
["testsToRun"][-1]
238 def check_for_intermittent(self
, options
):
239 "This method is used to check whether a test is an intermittent."
240 if self
.result
[options
.bisectChunk
] == "PASS":
242 "\t\tThe test %s passed." % self
.contents
["testsToRun"][0]
245 # loop is set to 1 to again run the single test.
246 self
.contents
["loop"] = 1
250 if self
.failcount
> 0:
251 # -1 is being returned as the test is intermittent, so no need to bisect
254 # If the test does not fail even once, then proceed to next chunk for bisection.
255 # loop is set to 2 to proceed on bisection.
256 self
.contents
["loop"] = 2
258 elif self
.result
[options
.bisectChunk
] == "FAIL":
260 "\t\tThe test %s failed." % self
.contents
["testsToRun"][0]
263 self
.contents
["loop"] = 1
265 # self.max_failures is the maximum number of times a test is allowed
266 # to fail to be called an intermittent. If a test fails more than
267 # limit set, it is a perma-fail.
268 if self
.failcount
< self
.max_failures
:
270 # -1 is being returned as the test is intermittent, so no need to bisect
276 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
277 "root cause for many of the above failures"
278 % self
.contents
["testsToRun"][0]
282 def print_summary(self
):
283 "This method is used to print the recorded summary."
284 print("Bisection summary:")
285 for line
in self
.summary
: