testing/mochitest/bisection.py

   1 from __future__ import absolute_import, division, print_function
   2
   3 import math
   4 import mozinfo
   5
   6
   7 class Bisect(object):
   8
   9     "Class for creating, bisecting and summarizing for --bisect-chunk option."
  10
  11     def __init__(self, harness):
  12         super(Bisect, self).__init__()
  13         self.summary = []
  14         self.contents = {}
  15         self.repeat = 10
  16         self.failcount = 0
  17         self.max_failures = 3
  18
  19     def setup(self, tests):
  20         """This method is used to initialize various variables that are required
  21         for test bisection"""
  22         status = 0
  23         self.contents.clear()
  24         # We need totalTests key in contents for sanity check
  25         self.contents["totalTests"] = tests
  26         self.contents["tests"] = tests
  27         self.contents["loop"] = 0
  28         return status
  29
  30     def reset(self, expectedError, result):
  31         """This method is used to initialize self.expectedError and self.result
  32         for each loop in runtests."""
  33         self.expectedError = expectedError
  34         self.result = result
  35
  36     def get_tests_for_bisection(self, options, tests):
  37         """Make a list of tests for bisection from a given list of tests"""
  38         bisectlist = []
  39         for test in tests:
  40             bisectlist.append(test)
  41             if test.endswith(options.bisectChunk):
  42                 break
  43
  44         return bisectlist
  45
  46     def pre_test(self, options, tests, status):
  47         """This method is used to call other methods for setting up variables and
  48         getting the list of tests for bisection."""
  49         if options.bisectChunk == "default":
  50             return tests
  51         # The second condition in 'if' is required to verify that the failing
  52         # test is the last one.
  53         elif "loop" not in self.contents or not self.contents["tests"][-1].endswith(
  54             options.bisectChunk
  55         ):
  56             tests = self.get_tests_for_bisection(options, tests)
  57             status = self.setup(tests)
  58
  59         return self.next_chunk_binary(options, status)
  60
  61     def post_test(self, options, expectedError, result):
  62         """This method is used to call other methods to summarize results and check whether a
  63         sanity check is done or not."""
  64         self.reset(expectedError, result)
  65         status = self.summarize_chunk(options)
  66         # Check whether sanity check has to be done. Also it is necessary to check whether
  67         # options.bisectChunk is present in self.expectedError as we do not want to run
  68         # if it is "default".
  69         if status == -1 and options.bisectChunk in self.expectedError:
  70             # In case we have a debug build, we don't want to run a sanity
  71             # check, will take too much time.
  72             if mozinfo.info["debug"]:
  73                 return status
  74
  75             testBleedThrough = self.contents["testsToRun"][0]
  76             tests = self.contents["totalTests"]
  77             tests.remove(testBleedThrough)
  78             # To make sure that the failing test is dependent on some other
  79             # test.
  80             if options.bisectChunk in testBleedThrough:
  81                 return status
  82
  83             status = self.setup(tests)
  84             self.summary.append("Sanity Check:")
  85
  86         return status
  87
  88     def next_chunk_reverse(self, options, status):
  89         "This method is used to bisect the tests in a reverse search fashion."
  90
  91         # Base Cases.
  92         if self.contents["loop"] <= 1:
  93             self.contents["testsToRun"] = self.contents["tests"]
  94             if self.contents["loop"] == 1:
  95                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
  96             self.contents["loop"] += 1
  97             return self.contents["testsToRun"]
  98
  99         if "result" in self.contents:
 100             if self.contents["result"] == "PASS":
 101                 chunkSize = self.contents["end"] - self.contents["start"]
 102                 self.contents["end"] = self.contents["start"] - 1
 103                 self.contents["start"] = self.contents["end"] - chunkSize
 104
 105             # self.contents['result'] will be expected error only if it fails.
 106             elif self.contents["result"] == "FAIL":
 107                 self.contents["tests"] = self.contents["testsToRun"]
 108                 status = 1  # for initializing
 109
 110         # initialize
 111         if status:
 112             totalTests = len(self.contents["tests"])
 113             chunkSize = int(math.ceil(totalTests / 10.0))
 114             self.contents["start"] = totalTests - chunkSize - 1
 115             self.contents["end"] = totalTests - 2
 116
 117         start = self.contents["start"]
 118         end = self.contents["end"] + 1
 119         self.contents["testsToRun"] = self.contents["tests"][start:end]
 120         self.contents["testsToRun"].append(self.contents["tests"][-1])
 121         self.contents["loop"] += 1
 122
 123         return self.contents["testsToRun"]
 124
 125     def next_chunk_binary(self, options, status):
 126         "This method is used to bisect the tests in a binary search fashion."
 127
 128         # Base cases.
 129         if self.contents["loop"] <= 1:
 130             self.contents["testsToRun"] = self.contents["tests"]
 131             if self.contents["loop"] == 1:
 132                 self.contents["testsToRun"] = [self.contents["tests"][-1]]
 133             self.contents["loop"] += 1
 134             return self.contents["testsToRun"]
 135
 136         # Initialize the contents dict.
 137         if status:
 138             totalTests = len(self.contents["tests"])
 139             self.contents["start"] = 0
 140             self.contents["end"] = totalTests - 2
 141
 142         # pylint --py3k W1619
 143         mid = (self.contents["start"] + self.contents["end"]) / 2
 144         if "result" in self.contents:
 145             if self.contents["result"] == "PASS":
 146                 self.contents["end"] = mid
 147
 148             elif self.contents["result"] == "FAIL":
 149                 self.contents["start"] = mid + 1
 150
 151         mid = (self.contents["start"] + self.contents["end"]) / 2
 152         start = mid + 1
 153         end = self.contents["end"] + 1
 154         self.contents["testsToRun"] = self.contents["tests"][start:end]
 155         if not self.contents["testsToRun"]:
 156             self.contents["testsToRun"].append(self.contents["tests"][mid])
 157         self.contents["testsToRun"].append(self.contents["tests"][-1])
 158         self.contents["loop"] += 1
 159
 160         return self.contents["testsToRun"]
 161
 162     def summarize_chunk(self, options):
 163         "This method is used summarize the results after the list of tests is run."
 164         if options.bisectChunk == "default":
 165             # if no expectedError that means all the tests have successfully
 166             # passed.
 167             if len(self.expectedError) == 0:
 168                 return -1
 169             options.bisectChunk = self.expectedError.keys()[0]
 170             self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
 171             return 0
 172
 173         # If options.bisectChunk is not in self.result then we need to move to
 174         # the next run.
 175         if options.bisectChunk not in self.result:
 176             return -1
 177
 178         self.summary.append("\tPass %d:" % self.contents["loop"])
 179         if len(self.contents["testsToRun"]) > 1:
 180             self.summary.append(
 181                 "\t\t%d test files(start,end,failing). [%s, %s, %s]"
 182                 % (
 183                     len(self.contents["testsToRun"]),
 184                     self.contents["testsToRun"][0],
 185                     self.contents["testsToRun"][-2],
 186                     self.contents["testsToRun"][-1],
 187                 )
 188             )
 189         else:
 190             self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0])
 191             return self.check_for_intermittent(options)
 192
 193         if self.result[options.bisectChunk] == "PASS":
 194             self.summary.append("\t\tno failures found.")
 195             if self.contents["loop"] == 1:
 196                 status = -1
 197             else:
 198                 self.contents["result"] = "PASS"
 199                 status = 0
 200
 201         elif self.result[options.bisectChunk] == "FAIL":
 202             if "expectedError" not in self.contents:
 203                 self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1])
 204                 self.contents["expectedError"] = self.expectedError[options.bisectChunk]
 205                 status = 0
 206
 207             elif (
 208                 self.expectedError[options.bisectChunk]
 209                 == self.contents["expectedError"]
 210             ):
 211                 self.summary.append(
 212                     "\t\t%s failed with expected error."
 213                     % self.contents["testsToRun"][-1]
 214                 )
 215                 self.contents["result"] = "FAIL"
 216                 status = 0
 217
 218                 # This code checks for test-bleedthrough. Should work for any
 219                 # algorithm.
 220                 numberOfTests = len(self.contents["testsToRun"])
 221                 if numberOfTests < 3:
 222                     # This means that only 2 tests are run. Since the last test
 223                     # is the failing test itself therefore the bleedthrough
 224                     # test is the first test
 225                     self.summary.append(
 226                         "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 227                         "root cause for many of the above failures"
 228                         % self.contents["testsToRun"][0]
 229                     )
 230                     status = -1
 231             else:
 232                 self.summary.append(
 233                     "\t\t%s failed with different error."
 234                     % self.contents["testsToRun"][-1]
 235                 )
 236                 status = -1
 237
 238         return status
 239
 240     def check_for_intermittent(self, options):
 241         "This method is used to check whether a test is an intermittent."
 242         if self.result[options.bisectChunk] == "PASS":
 243             self.summary.append(
 244                 "\t\tThe test %s passed." % self.contents["testsToRun"][0]
 245             )
 246             if self.repeat > 0:
 247                 # loop is set to 1 to again run the single test.
 248                 self.contents["loop"] = 1
 249                 self.repeat -= 1
 250                 return 0
 251             else:
 252                 if self.failcount > 0:
 253                     # -1 is being returned as the test is intermittent, so no need to bisect
 254                     # further.
 255                     return -1
 256                 # If the test does not fail even once, then proceed to next chunk for bisection.
 257                 # loop is set to 2 to proceed on bisection.
 258                 self.contents["loop"] = 2
 259                 return 1
 260         elif self.result[options.bisectChunk] == "FAIL":
 261             self.summary.append(
 262                 "\t\tThe test %s failed." % self.contents["testsToRun"][0]
 263             )
 264             self.failcount += 1
 265             self.contents["loop"] = 1
 266             self.repeat -= 1
 267             # self.max_failures is the maximum number of times a test is allowed
 268             # to fail to be called an intermittent. If a test fails more than
 269             # limit set, it is a perma-fail.
 270             if self.failcount < self.max_failures:
 271                 if self.repeat == 0:
 272                     # -1 is being returned as the test is intermittent, so no need to bisect
 273                     # further.
 274                     return -1
 275                 return 0
 276             else:
 277                 self.summary.append(
 278                     "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
 279                     "root cause for many of the above failures"
 280                     % self.contents["testsToRun"][0]
 281                 )
 282                 return -1
 283
 284     def print_summary(self):
 285         "This method is used to print the recorded summary."
 286         print("Bisection summary:")
 287         for line in self.summary:
 288             print(line)