testing/mochitest/bisection.py

   1 import math
   2
   3 import mozinfo
   4
   5
   6 class Bisect(object):
   7     "Class for creating, bisecting and summarizing for --bisect-chunk option."
   8
   9     def __init__(self, harness):
  10         super(Bisect, self).__init__()
  11         self.summary = []
  12         self.contents = {}
  13         self.repeat = 10
  14         self.failcount = 0
  15         self.max_failures = 3
  16
  17     def setup(self, tests):
  18         """This method is used to initialize various variables that are required
  19         for test bisection"""
  20         status = 0
  21         self.contents.clear()
  22         # We need totalTests key in contents for sanity check
  23         self.contents["totalTests"] = tests
  24         self.contents["tests"] = tests
  25         self.contents["loop"] = 0
  26         return status
  27
  28     def reset(self, expectedError, result):
  29         """This method is used to initialize self.expectedError and self.result
  30         for each loop in runtests."""
  31         self.expectedError = expectedError
  32         self.result = result
  33
  34     def get_tests_for_bisection(self, options, tests):
  35         """Make a list of tests for bisection from a given list of tests"""
  36         bisectlist = []
  37         for test in tests:
  38             bisectlist.append(test)
  39             if test.endswith(options.bisectChunk):
  40                 break
  41
  42         return bisectlist
  43
  44     def pre_test(self, options, tests, status):
  45         """This method is used to call other methods for setting up variables and
  46         getting the list of tests for bisection."""
  47         if options.bisectChunk == "default":
  48             return tests
  49         # The second condition in 'if' is required to verify that the failing
  50         # test is the last one.
  51         elif "loop" not in self.contents or not self.contents["tests"][-1].endswith(
  52             options.bisectChunk
  53         ):
  54             tests = self.get_tests_for_bisection(options, tests)
  55             status = self.setup(tests)
  56
  57         return self.next_chunk_binary(options, status)
  58
  59     def post_test(self, options, expectedError, result):
  60         """This method is used to call other methods to summarize results and check whether a
  61         sanity check is done or not."""
  62         self.reset(expectedError, result)
  63         status = self.summarize_chunk(options)
  64         # Check whether sanity check has to be done. Also it is necessary to check whether
  65         # options.bisectChunk is present in self.expectedError as we do not want to run
  66         # if it is "default".
  67         if status == -1 and options.bisectChunk in self.expectedError:
  68             # In case we have a debug build, we don't want to run a sanity
  69             # check, will take too much time.
  70             if mozinfo.info["debug"]:
  71                 return status
  72
  73             testBleedThrough = self.contents["testsToRun"][0]
  74             tests = self.contents["totalTests"]
  75             tests.remove(testBleedThrough)
  76             # To make sure that the failing test is dependent on some other
  77             # test.
  78             if options.bisectChunk in testBleedThrough:
  79                 return status
  80
  81             status = self.setup(tests)
  82             self.summary.append("Sanity Check:")
  83
  84         return status
  85
  86     def next_chunk_reverse(self, options, status):
  87         "This method is used to bisect the tests in a reverse search fashion."
  88
  89         # Base Cases.
  90         if self.contents["loop"] <= 1:
  91             self.contents["testsToRun"] = self.contents["tests"]
  92             if self.contents["loop"] == 1:
  93                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
  94             self.contents["loop"] += 1
  95             return self.contents["testsToRun"]
  96
  97         if "result" in self.contents:
  98             if self.contents["result"] == "PASS":
  99                 chunkSize = self.contents["end"] - self.contents["start"]
 100                 self.contents["end"] = self.contents["start"] - 1
 101                 self.contents["start"] = self.contents["end"] - chunkSize
 102
 103             # self.contents['result'] will be expected error only if it fails.
 104             elif self.contents["result"] == "FAIL":
 105                 self.contents["tests"] = self.contents["testsToRun"]
 106                 status = 1  # for initializing
 107
 108         # initialize
 109         if status:
 110             totalTests = len(self.contents["tests"])
 111             chunkSize = int(math.ceil(totalTests / 10.0))
 112             self.contents["start"] = totalTests - chunkSize - 1
 113             self.contents["end"] = totalTests - 2
 114
 115         start = self.contents["start"]
 116         end = self.contents["end"] + 1
 117         self.contents["testsToRun"] = self.contents["tests"][start:end]
 118         self.contents["testsToRun"].append(self.contents["tests"][-1])
 119         self.contents["loop"] += 1
 120
 121         return self.contents["testsToRun"]
 122
 123     def next_chunk_binary(self, options, status):
 124         "This method is used to bisect the tests in a binary search fashion."
 125
 126         # Base cases.
 127         if self.contents["loop"] <= 1:
 128             self.contents["testsToRun"] = self.contents["tests"]
 129             if self.contents["loop"] == 1:
 130                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
 131             self.contents["loop"] += 1
 132             return self.contents["testsToRun"]
 133
 134         # Initialize the contents dict.
 135         if status:
 136             totalTests = len(self.contents["tests"])
 137             self.contents["start"] = 0
 138             self.contents["end"] = totalTests - 2
 139
 140         # pylint --py3k W1619
 141         mid = (self.contents["start"] + self.contents["end"]) / 2
 142         if "result" in self.contents:
 143             if self.contents["result"] == "PASS":
 144                 self.contents["end"] = mid
 145
 146             elif self.contents["result"] == "FAIL":
 147                 self.contents["start"] = mid + 1
 148
 149         mid = (self.contents["start"] + self.contents["end"]) / 2
 150         start = mid + 1
 151         end = self.contents["end"] + 1
 152         self.contents["testsToRun"] = self.contents["tests"][start:end]
 153         if not self.contents["testsToRun"]:
 154             self.contents["testsToRun"].append(self.contents["tests"][mid])
 155         self.contents["testsToRun"].append(self.contents["tests"][-1])
 156         self.contents["loop"] += 1
 157
 158         return self.contents["testsToRun"]
 159
 160     def summarize_chunk(self, options):
 161         "This method is used summarize the results after the list of tests is run."
 162         if options.bisectChunk == "default":
 163             # if no expectedError that means all the tests have successfully
 164             # passed.
 165             if len(self.expectedError) == 0:
 166                 return -1
 167             options.bisectChunk = self.expectedError.keys()[0]
 168             self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
 169             return 0
 170
 171         # If options.bisectChunk is not in self.result then we need to move to
 172         # the next run.
 173         if options.bisectChunk not in self.result:
 174             return -1
 175
 176         self.summary.append("\tPass %d:" % self.contents["loop"])
 177         if len(self.contents["testsToRun"]) > 1:
 178             self.summary.append(
 179                 "\t\t%d test files(start,end,failing). [%s, %s, %s]"
 180                 % (
 181                     len(self.contents["testsToRun"]),
 182                     self.contents["testsToRun"][0],
 183                     self.contents["testsToRun"][-2],
 184                     self.contents["testsToRun"][-1],
 185                 )
 186             )
 187         else:
 188             self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0])
 189             return self.check_for_intermittent(options)
 190
 191         if self.result[options.bisectChunk] == "PASS":
 192             self.summary.append("\t\tno failures found.")
 193             if self.contents["loop"] == 1:
 194                 status = -1
 195             else:
 196                 self.contents["result"] = "PASS"
 197                 status = 0
 198
 199         elif self.result[options.bisectChunk] == "FAIL":
 200             if "expectedError" not in self.contents:
 201                 self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1])
 202                 self.contents["expectedError"] = self.expectedError[options.bisectChunk]
 203                 status = 0
 204
 205             elif (
 206                 self.expectedError[options.bisectChunk]
 207                 == self.contents["expectedError"]
 208             ):
 209                 self.summary.append(
 210                     "\t\t%s failed with expected error."
 211                     % self.contents["testsToRun"][-1]
 212                 )
 213                 self.contents["result"] = "FAIL"
 214                 status = 0
 215
 216                 # This code checks for test-bleedthrough. Should work for any
 217                 # algorithm.
 218                 numberOfTests = len(self.contents["testsToRun"])
 219                 if numberOfTests < 3:
 220                     # This means that only 2 tests are run. Since the last test
 221                     # is the failing test itself therefore the bleedthrough
 222                     # test is the first test
 223                     self.summary.append(
 224                         "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 225                         "root cause for many of the above failures"
 226                         % self.contents["testsToRun"][0]
 227                     )
 228                     status = -1
 229             else:
 230                 self.summary.append(
 231                     "\t\t%s failed with different error."
 232                     % self.contents["testsToRun"][-1]
 233                 )
 234                 status = -1
 235
 236         return status
 237
 238     def check_for_intermittent(self, options):
 239         "This method is used to check whether a test is an intermittent."
 240         if self.result[options.bisectChunk] == "PASS":
 241             self.summary.append(
 242                 "\t\tThe test %s passed." % self.contents["testsToRun"][0]
 243             )
 244             if self.repeat > 0:
 245                 # loop is set to 1 to again run the single test.
 246                 self.contents["loop"] = 1
 247                 self.repeat -= 1
 248                 return 0
 249             else:
 250                 if self.failcount > 0:
 251                     # -1 is being returned as the test is intermittent, so no need to bisect
 252                     # further.
 253                     return -1
 254                 # If the test does not fail even once, then proceed to next chunk for bisection.
 255                 # loop is set to 2 to proceed on bisection.
 256                 self.contents["loop"] = 2
 257                 return 1
 258         elif self.result[options.bisectChunk] == "FAIL":
 259             self.summary.append(
 260                 "\t\tThe test %s failed." % self.contents["testsToRun"][0]
 261             )
 262             self.failcount += 1
 263             self.contents["loop"] = 1
 264             self.repeat -= 1
 265             # self.max_failures is the maximum number of times a test is allowed
 266             # to fail to be called an intermittent. If a test fails more than
 267             # limit set, it is a perma-fail.
 268             if self.failcount < self.max_failures:
 269                 if self.repeat == 0:
 270                     # -1 is being returned as the test is intermittent, so no need to bisect
 271                     # further.
 272                     return -1
 273                 return 0
 274             else:
 275                 self.summary.append(
 276                     "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 277                     "root cause for many of the above failures"
 278                     % self.contents["testsToRun"][0]
 279                 )
 280                 return -1
 281
 282     def print_summary(self):
 283         "This method is used to print the recorded summary."
 284         print("Bisection summary:")
 285         for line in self.summary:
 286             print(line)