testing/mochitest/bisection.py

   1 import math
   2
   3 import mozinfo
   4
   5
   6 class Bisect(object):
   7
   8     "Class for creating, bisecting and summarizing for --bisect-chunk option."
   9
  10     def __init__(self, harness):
  11         super(Bisect, self).__init__()
  12         self.summary = []
  13         self.contents = {}
  14         self.repeat = 10
  15         self.failcount = 0
  16         self.max_failures = 3
  17
  18     def setup(self, tests):
  19         """This method is used to initialize various variables that are required
  20         for test bisection"""
  21         status = 0
  22         self.contents.clear()
  23         # We need totalTests key in contents for sanity check
  24         self.contents["totalTests"] = tests
  25         self.contents["tests"] = tests
  26         self.contents["loop"] = 0
  27         return status
  28
  29     def reset(self, expectedError, result):
  30         """This method is used to initialize self.expectedError and self.result
  31         for each loop in runtests."""
  32         self.expectedError = expectedError
  33         self.result = result
  34
  35     def get_tests_for_bisection(self, options, tests):
  36         """Make a list of tests for bisection from a given list of tests"""
  37         bisectlist = []
  38         for test in tests:
  39             bisectlist.append(test)
  40             if test.endswith(options.bisectChunk):
  41                 break
  42
  43         return bisectlist
  44
  45     def pre_test(self, options, tests, status):
  46         """This method is used to call other methods for setting up variables and
  47         getting the list of tests for bisection."""
  48         if options.bisectChunk == "default":
  49             return tests
  50         # The second condition in 'if' is required to verify that the failing
  51         # test is the last one.
  52         elif "loop" not in self.contents or not self.contents["tests"][-1].endswith(
  53             options.bisectChunk
  54         ):
  55             tests = self.get_tests_for_bisection(options, tests)
  56             status = self.setup(tests)
  57
  58         return self.next_chunk_binary(options, status)
  59
  60     def post_test(self, options, expectedError, result):
  61         """This method is used to call other methods to summarize results and check whether a
  62         sanity check is done or not."""
  63         self.reset(expectedError, result)
  64         status = self.summarize_chunk(options)
  65         # Check whether sanity check has to be done. Also it is necessary to check whether
  66         # options.bisectChunk is present in self.expectedError as we do not want to run
  67         # if it is "default".
  68         if status == -1 and options.bisectChunk in self.expectedError:
  69             # In case we have a debug build, we don't want to run a sanity
  70             # check, will take too much time.
  71             if mozinfo.info["debug"]:
  72                 return status
  73
  74             testBleedThrough = self.contents["testsToRun"][0]
  75             tests = self.contents["totalTests"]
  76             tests.remove(testBleedThrough)
  77             # To make sure that the failing test is dependent on some other
  78             # test.
  79             if options.bisectChunk in testBleedThrough:
  80                 return status
  81
  82             status = self.setup(tests)
  83             self.summary.append("Sanity Check:")
  84
  85         return status
  86
  87     def next_chunk_reverse(self, options, status):
  88         "This method is used to bisect the tests in a reverse search fashion."
  89
  90         # Base Cases.
  91         if self.contents["loop"] <= 1:
  92             self.contents["testsToRun"] = self.contents["tests"]
  93             if self.contents["loop"] == 1:
  94                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
  95             self.contents["loop"] += 1
  96             return self.contents["testsToRun"]
  97
  98         if "result" in self.contents:
  99             if self.contents["result"] == "PASS":
 100                 chunkSize = self.contents["end"] - self.contents["start"]
 101                 self.contents["end"] = self.contents["start"] - 1
 102                 self.contents["start"] = self.contents["end"] - chunkSize
 103
 104             # self.contents['result'] will be expected error only if it fails.
 105             elif self.contents["result"] == "FAIL":
 106                 self.contents["tests"] = self.contents["testsToRun"]
 107                 status = 1  # for initializing
 108
 109         # initialize
 110         if status:
 111             totalTests = len(self.contents["tests"])
 112             chunkSize = int(math.ceil(totalTests / 10.0))
 113             self.contents["start"] = totalTests - chunkSize - 1
 114             self.contents["end"] = totalTests - 2
 115
 116         start = self.contents["start"]
 117         end = self.contents["end"] + 1
 118         self.contents["testsToRun"] = self.contents["tests"][start:end]
 119         self.contents["testsToRun"].append(self.contents["tests"][-1])
 120         self.contents["loop"] += 1
 121
 122         return self.contents["testsToRun"]
 123
 124     def next_chunk_binary(self, options, status):
 125         "This method is used to bisect the tests in a binary search fashion."
 126
 127         # Base cases.
 128         if self.contents["loop"] <= 1:
 129             self.contents["testsToRun"] = self.contents["tests"]
 130             if self.contents["loop"] == 1:
 131                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
 132             self.contents["loop"] += 1
 133             return self.contents["testsToRun"]
 134
 135         # Initialize the contents dict.
 136         if status:
 137             totalTests = len(self.contents["tests"])
 138             self.contents["start"] = 0
 139             self.contents["end"] = totalTests - 2
 140
 141         # pylint --py3k W1619
 142         mid = (self.contents["start"] + self.contents["end"]) / 2
 143         if "result" in self.contents:
 144             if self.contents["result"] == "PASS":
 145                 self.contents["end"] = mid
 146
 147             elif self.contents["result"] == "FAIL":
 148                 self.contents["start"] = mid + 1
 149
 150         mid = (self.contents["start"] + self.contents["end"]) / 2
 151         start = mid + 1
 152         end = self.contents["end"] + 1
 153         self.contents["testsToRun"] = self.contents["tests"][start:end]
 154         if not self.contents["testsToRun"]:
 155             self.contents["testsToRun"].append(self.contents["tests"][mid])
 156         self.contents["testsToRun"].append(self.contents["tests"][-1])
 157         self.contents["loop"] += 1
 158
 159         return self.contents["testsToRun"]
 160
 161     def summarize_chunk(self, options):
 162         "This method is used summarize the results after the list of tests is run."
 163         if options.bisectChunk == "default":
 164             # if no expectedError that means all the tests have successfully
 165             # passed.
 166             if len(self.expectedError) == 0:
 167                 return -1
 168             options.bisectChunk = self.expectedError.keys()[0]
 169             self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
 170             return 0
 171
 172         # If options.bisectChunk is not in self.result then we need to move to
 173         # the next run.
 174         if options.bisectChunk not in self.result:
 175             return -1
 176
 177         self.summary.append("\tPass %d:" % self.contents["loop"])
 178         if len(self.contents["testsToRun"]) > 1:
 179             self.summary.append(
 180                 "\t\t%d test files(start,end,failing). [%s, %s, %s]"
 181                 % (
 182                     len(self.contents["testsToRun"]),
 183                     self.contents["testsToRun"][0],
 184                     self.contents["testsToRun"][-2],
 185                     self.contents["testsToRun"][-1],
 186                 )
 187             )
 188         else:
 189             self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0])
 190             return self.check_for_intermittent(options)
 191
 192         if self.result[options.bisectChunk] == "PASS":
 193             self.summary.append("\t\tno failures found.")
 194             if self.contents["loop"] == 1:
 195                 status = -1
 196             else:
 197                 self.contents["result"] = "PASS"
 198                 status = 0
 199
 200         elif self.result[options.bisectChunk] == "FAIL":
 201             if "expectedError" not in self.contents:
 202                 self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1])
 203                 self.contents["expectedError"] = self.expectedError[options.bisectChunk]
 204                 status = 0
 205
 206             elif (
 207                 self.expectedError[options.bisectChunk]
 208                 == self.contents["expectedError"]
 209             ):
 210                 self.summary.append(
 211                     "\t\t%s failed with expected error."
 212                     % self.contents["testsToRun"][-1]
 213                 )
 214                 self.contents["result"] = "FAIL"
 215                 status = 0
 216
 217                 # This code checks for test-bleedthrough. Should work for any
 218                 # algorithm.
 219                 numberOfTests = len(self.contents["testsToRun"])
 220                 if numberOfTests < 3:
 221                     # This means that only 2 tests are run. Since the last test
 222                     # is the failing test itself therefore the bleedthrough
 223                     # test is the first test
 224                     self.summary.append(
 225                         "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 226                         "root cause for many of the above failures"
 227                         % self.contents["testsToRun"][0]
 228                     )
 229                     status = -1
 230             else:
 231                 self.summary.append(
 232                     "\t\t%s failed with different error."
 233                     % self.contents["testsToRun"][-1]
 234                 )
 235                 status = -1
 236
 237         return status
 238
 239     def check_for_intermittent(self, options):
 240         "This method is used to check whether a test is an intermittent."
 241         if self.result[options.bisectChunk] == "PASS":
 242             self.summary.append(
 243                 "\t\tThe test %s passed." % self.contents["testsToRun"][0]
 244             )
 245             if self.repeat > 0:
 246                 # loop is set to 1 to again run the single test.
 247                 self.contents["loop"] = 1
 248                 self.repeat -= 1
 249                 return 0
 250             else:
 251                 if self.failcount > 0:
 252                     # -1 is being returned as the test is intermittent, so no need to bisect
 253                     # further.
 254                     return -1
 255                 # If the test does not fail even once, then proceed to next chunk for bisection.
 256                 # loop is set to 2 to proceed on bisection.
 257                 self.contents["loop"] = 2
 258                 return 1
 259         elif self.result[options.bisectChunk] == "FAIL":
 260             self.summary.append(
 261                 "\t\tThe test %s failed." % self.contents["testsToRun"][0]
 262             )
 263             self.failcount += 1
 264             self.contents["loop"] = 1
 265             self.repeat -= 1
 266             # self.max_failures is the maximum number of times a test is allowed
 267             # to fail to be called an intermittent. If a test fails more than
 268             # limit set, it is a perma-fail.
 269             if self.failcount < self.max_failures:
 270                 if self.repeat == 0:
 271                     # -1 is being returned as the test is intermittent, so no need to bisect
 272                     # further.
 273                     return -1
 274                 return 0
 275             else:
 276                 self.summary.append(
 277                     "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 278                     "root cause for many of the above failures"
 279                     % self.contents["testsToRun"][0]
 280                 )
 281                 return -1
 282
 283     def print_summary(self):
 284         "This method is used to print the recorded summary."
 285         print("Bisection summary:")
 286         for line in self.summary:
 287             print(line)