tools/bisect-perf-regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import unittest
   6
   7 from auto_bisect import source_control as source_control_module
   8
   9 # Special import necessary because filename contains dash characters.
  10 bisect_perf_module = __import__('bisect-perf-regression')
  11
  12 # Sample output for a performance test used in the results parsing tests below.
  13 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count
  14 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb
  15 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb
  16 RESULT telemetry_page_measurement_results: num_failed= 0 count
  17 RESULT telemetry_page_measurement_results: num_errored= 0 count
  18 *RESULT Total: Total_ref= %(value)s
  19 """
  20
  21
  22 # Some private methods of the bisect-perf-regression module are tested below.
  23 # pylint: disable=W0212
  24 class BisectPerfRegressionTest(unittest.TestCase):
  25   """Test case for top-level functions in the bisect-perf-regrssion module."""
  26
  27   def setUp(self):
  28     """Sets up the test environment before each test method."""
  29     pass
  30
  31   def tearDown(self):
  32     """Cleans up the test environment after each test method."""
  33     pass
  34
  35   def testConfidenceScoreHigh(self):
  36     """Tests the confidence calculation."""
  37     bad_values = [[0, 1, 1], [1, 2, 2]]
  38     good_values = [[1, 2, 2], [3, 3, 4]]
  39     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  40     self.assertEqual(95.0, confidence)
  41
  42   def testConfidenceScoreNotSoHigh(self):
  43     """Tests the confidence calculation."""
  44     bad_values = [[0, 1, 1], [1, 2, 2]]
  45     good_values = [[1, 1, 1], [3, 3, 4]]
  46     # The good and bad groups are closer together than in the above test,
  47     # so the confidence that they're different is a little lower.
  48     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  49     self.assertEqual(80.0, confidence)
  50
  51   def testConfidenceScoreZero(self):
  52     """Tests the confidence calculation when it's expected to be 0."""
  53     bad_values = [[4, 5], [7, 6], [8, 7]]
  54     good_values = [[8, 7], [6, 7], [5, 4]]
  55     # The good and bad sets contain the same values, so the confidence that
  56     # they're different should be zero.
  57     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  58     self.assertEqual(0.0, confidence)
  59
  60   def testConfidenceScoreVeryHigh(self):
  61     """Tests the confidence calculation when it's expected to be high."""
  62     bad_values = [[1, 1], [1, 1]]
  63     good_values = [[1.2, 1.2], [1.2, 1.2]]
  64     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  65     self.assertEqual(99.9, confidence)
  66
  67   def testConfidenceScoreImbalance(self):
  68     """Tests the confidence calculation one set of numbers is small."""
  69     bad_values = [[1.1, 1.2], [1.1, 1.2], [1.0, 1.3], [1.2, 1.3]]
  70     good_values = [[1.4]]
  71     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  72     self.assertEqual(80.0, confidence)
  73
  74   def testConfidenceScoreImbalance(self):
  75     """Tests the confidence calculation one set of numbers is empty."""
  76     bad_values = [[1.1, 1.2], [1.1, 1.2], [1.0, 1.3], [1.2, 1.3]]
  77     good_values = []
  78     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  79     self.assertEqual(0.0, confidence)
  80
  81   def testConfidenceScoreFunctionalTestResultsInconsistent(self):
  82     """Tests the confidence calculation when the numbers are just 0 and 1."""
  83     bad_values = [[1], [1], [0], [1], [1], [1], [0], [1]]
  84     good_values = [[0], [0], [1], [0], [1], [0]]
  85     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  86     self.assertEqual(80.0, confidence)
  87
  88   def testConfidenceScoreFunctionalTestResultsConsistent(self):
  89     """Tests the confidence calculation when the numbers are 0 and 1."""
  90     bad_values = [[1], [1], [1], [1], [1], [1], [1], [1]]
  91     good_values = [[0], [0], [0], [0], [0], [0]]
  92     confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values)
  93     self.assertEqual(99.9, confidence)
  94
  95   def testParseDEPSStringManually(self):
  96     """Tests DEPS parsing."""
  97     deps_file_contents = """
  98     vars = {
  99         'ffmpeg_hash':
 100              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 101         'webkit_url':
 102              'https://chromium.googlesource.com/chromium/blink.git',
 103         'git_url':
 104              'https://chromium.googlesource.com',
 105         'webkit_rev':
 106              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 107         'angle_revision':
 108              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 109     }"""
 110
 111     # Should only expect svn/git revisions to come through, and urls should be
 112     # filtered out.
 113     expected_vars_dict = {
 114         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 115         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 116         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 117     }
 118     vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually(
 119         deps_file_contents)
 120     self.assertEqual(vars_dict, expected_vars_dict)
 121
 122   def testTryParseResultValuesFromOutputWithSingleValue(self):
 123     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 124     metrics = ['Total', 'Total_ref']
 125     self.assertEqual(
 126         [66.88], bisect_perf_module._TryParseResultValuesFromOutput(
 127             metrics, RESULTS_OUTPUT % {'value': '66.88 kb'}))
 128     self.assertEqual(
 129         [66.88], bisect_perf_module._TryParseResultValuesFromOutput(
 130             metrics, RESULTS_OUTPUT % {'value': '66.88kb'}))
 131     self.assertEqual(
 132         [66.88], bisect_perf_module._TryParseResultValuesFromOutput(
 133             metrics, RESULTS_OUTPUT % {'value': ' 66.88 '}))
 134     self.assertEqual(
 135         [-66.88], bisect_perf_module._TryParseResultValuesFromOutput(
 136             metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'}))
 137     self.assertEqual(
 138         [66], bisect_perf_module._TryParseResultValuesFromOutput(
 139             metrics, RESULTS_OUTPUT % {'value': '66 kb'}))
 140     self.assertEqual(
 141         [.66], bisect_perf_module._TryParseResultValuesFromOutput(
 142             metrics, RESULTS_OUTPUT % {'value': '.66 kb'}))
 143     self.assertEqual(
 144         [], bisect_perf_module._TryParseResultValuesFromOutput(
 145             metrics, RESULTS_OUTPUT % {'value': '. kb'}))
 146     self.assertEqual(
 147         [], bisect_perf_module._TryParseResultValuesFromOutput(
 148             metrics, RESULTS_OUTPUT % {'value': 'aaa kb'}))
 149
 150   def testTryParseResultValuesFromOutputWithMulitValue(self):
 151     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 152     metrics = ['Total', 'Total_ref']
 153     self.assertEqual(
 154         [66.88], bisect_perf_module._TryParseResultValuesFromOutput(
 155             metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'}))
 156     self.assertEqual(
 157         [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput(
 158             metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'}))
 159     self.assertEqual(
 160         [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput(
 161             metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'}))
 162     self.assertEqual(
 163         [-66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput(
 164             metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'}))
 165     self.assertEqual(
 166         [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput(
 167             metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'}))
 168     self.assertEqual(
 169         [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput(
 170             metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'}))
 171     self.assertEqual(
 172         [.66, .99], bisect_perf_module._TryParseResultValuesFromOutput(
 173             metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'}))
 174     self.assertEqual(
 175         [], bisect_perf_module._TryParseResultValuesFromOutput(
 176             metrics, RESULTS_OUTPUT % {'value': '[] kb'}))
 177     self.assertEqual(
 178         [], bisect_perf_module._TryParseResultValuesFromOutput(
 179             metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'}))
 180
 181   def testTryParseResultValuesFromOutputWithMeanStd(self):
 182     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 183     metrics = ['Total', 'Total_ref']
 184     self.assertEqual(
 185         [33.22], bisect_perf_module._TryParseResultValuesFromOutput(
 186             metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'}))
 187     self.assertEqual(
 188         [33.22], bisect_perf_module._TryParseResultValuesFromOutput(
 189             metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'}))
 190     self.assertEqual(
 191         [33.22], bisect_perf_module._TryParseResultValuesFromOutput(
 192             metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'}))
 193     self.assertEqual(
 194         [33.22], bisect_perf_module._TryParseResultValuesFromOutput(
 195             metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'}))
 196     self.assertEqual(
 197         [-33.22], bisect_perf_module._TryParseResultValuesFromOutput(
 198             metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'}))
 199     self.assertEqual(
 200         [22], bisect_perf_module._TryParseResultValuesFromOutput(
 201             metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'}))
 202     self.assertEqual(
 203         [.22], bisect_perf_module._TryParseResultValuesFromOutput(
 204             metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'}))
 205     self.assertEqual(
 206         [], bisect_perf_module._TryParseResultValuesFromOutput(
 207             metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'}))
 208     self.assertEqual(
 209         [], bisect_perf_module._TryParseResultValuesFromOutput(
 210             metrics, RESULTS_OUTPUT % {'value': '{}kb'}))
 211     self.assertEqual(
 212         [], bisect_perf_module._TryParseResultValuesFromOutput(
 213             metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'}))
 214
 215   def testGetCompatibleCommand(self):
 216     bisect_options = bisect_perf_module.BisectOptions()
 217     bisect_options.output_buildbot_annotations = None
 218     source_control = source_control_module.DetermineAndCreateSourceControl(
 219         bisect_options)
 220     bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
 221         source_control, bisect_options)
 222     bisect_instance.opts.target_platform = 'android'
 223     depot = 'chromium'
 224     # android-chrome-shell -> android-chromium-testshell
 225     revision = 274857
 226     git_revision = bisect_instance.source_control.ResolveToRevision(
 227         revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100)
 228     command = ('tools/perf/run_benchmark -v '
 229                '--browser=android-chrome-shell page_cycler.intl_ja_zh')
 230     expected_command = ('tools/perf/run_benchmark -v --browser='
 231                         'android-chromium-testshell page_cycler.intl_ja_zh')
 232     self.assertEqual(
 233         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 234         expected_command)
 235
 236     # android-chromium-testshell -> android-chromium-testshell
 237     revision = 274858
 238     git_revision = bisect_instance.source_control.ResolveToRevision(
 239         revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100)
 240     command = ('tools/perf/run_benchmark -v '
 241                '--browser=android-chromium-testshell page_cycler.intl_ja_zh')
 242     expected_command = ('tools/perf/run_benchmark -v --browser='
 243                         'android-chromium-testshell page_cycler.intl_ja_zh')
 244     self.assertEqual(
 245         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 246         expected_command)
 247
 248     # android-chromium-testshell -> android-chrome-shell
 249     revision = 276628
 250     git_revision = bisect_instance.source_control.ResolveToRevision(
 251         revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100)
 252     command = ('tools/perf/run_benchmark -v '
 253                '--browser=android-chromium-testshell page_cycler.intl_ja_zh')
 254     expected_command = ('tools/perf/run_benchmark -v --browser='
 255                         'android-chrome-shell page_cycler.intl_ja_zh')
 256     self.assertEqual(
 257         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 258         expected_command)
 259     # android-chrome-shell -> android-chrome-shell
 260     command = ('tools/perf/run_benchmark -v '
 261                '--browser=android-chrome-shell page_cycler.intl_ja_zh')
 262     expected_command = ('tools/perf/run_benchmark -v --browser='
 263                         'android-chrome-shell page_cycler.intl_ja_zh')
 264     self.assertEqual(
 265         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 266         expected_command)
 267     # Not android platform
 268     bisect_instance.opts.target_platform = 'chromium'
 269     command = ('tools/perf/run_benchmark -v '
 270                '--browser=release page_cycler.intl_ja_zh')
 271     expected_command = ('tools/perf/run_benchmark -v --browser='
 272                         'release page_cycler.intl_ja_zh')
 273     self.assertEqual(
 274         bisect_instance.GetCompatibleCommand(command, git_revision, depot),
 275         expected_command)
 276
 277
 278 if __name__ == '__main__':
 279   unittest.main()