tools/auto_bisect/bisect_perf_regression_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import shutil
   8 import sys
   9 import unittest
  10
  11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
  12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
  13
  14 import bisect_perf_regression
  15 import bisect_utils
  16 import mock
  17 import source_control
  18
  19
  20 # Regression confidence: 0%
  21 CLEAR_NON_REGRESSION = [
  22     # Mean: 30.223 Std. Dev.: 11.383
  23     [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
  24      [19.552], [21.954], [38.573], [38.839], [38.965], [40.007], [40.572],
  25      [41.491], [42.002], [42.33], [43.109], [43.238]],
  26     # Mean: 34.76 Std. Dev.: 11.516
  27     [[16.426], [17.347], [20.593], [21.177], [22.791], [27.843], [28.383],
  28      [28.46], [29.143], [40.058], [40.303], [40.558], [41.918], [42.44],
  29      [45.223], [46.494], [50.002], [50.625], [50.839]]
  30 ]
  31 # Regression confidence: ~ 90%
  32 ALMOST_REGRESSION = [
  33     # Mean: 30.042 Std. Dev.: 2.002
  34     [[26.146], [28.04], [28.053], [28.074], [28.168], [28.209], [28.471],
  35      [28.652], [28.664], [30.862], [30.973], [31.002], [31.897], [31.929],
  36      [31.99], [32.214], [32.323], [32.452], [32.696]],
  37     # Mean: 33.008 Std. Dev.: 4.265
  38     [[34.963], [30.741], [39.677], [39.512], [34.314], [31.39], [34.361],
  39      [25.2], [30.489], [29.434]]
  40 ]
  41 # Regression confidence: ~ 98%
  42 BARELY_REGRESSION = [
  43     # Mean: 28.828 Std. Dev.: 1.993
  44     [[26.96], [27.605], [27.768], [27.829], [28.006], [28.206], [28.393],
  45      [28.911], [28.933], [30.38], [30.462], [30.808], [31.74], [31.805],
  46      [31.899], [32.077], [32.454], [32.597], [33.155]],
  47     # Mean: 31.156 Std. Dev.: 1.980
  48     [[28.729], [29.112], [29.258], [29.454], [29.789], [30.036], [30.098],
  49      [30.174], [30.534], [32.285], [32.295], [32.552], [32.572], [32.967],
  50      [33.165], [33.403], [33.588], [33.744], [34.147], [35.84]]
  51 ]
  52 # Regression confidence: 99.5%
  53 CLEAR_REGRESSION = [
  54     # Mean: 30.254 Std. Dev.: 2.987
  55     [[26.494], [26.621], [26.701], [26.997], [26.997], [27.05], [27.37],
  56      [27.488], [27.556], [31.846], [32.192], [32.21], [32.586], [32.596],
  57      [32.618], [32.95], [32.979], [33.421], [33.457], [34.97]],
  58     # Mean: 33.190 Std. Dev.: 2.972
  59     [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
  60      [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
  61      [35.934], [36.233], [36.41], [36.947], [37.982]]
  62 ]
  63
  64 # Regression confidence > 95%, taken from: crbug.com/434318
  65 # Specifically from Builder android_nexus10_perf_bisect Build #1198
  66 MULTIPLE_VALUES = [
  67     [
  68         [18.916000,22.371000,8.527000,5.877000,5.407000,9.476000,8.100000,
  69          5.334000,4.507000,4.842000,8.485000,8.308000,27.490000,4.560000,
  70          4.804000,23.068000,17.577000,17.346000,26.738000,60.330000,32.307000,
  71          5.468000,27.803000,27.373000,17.823000,5.158000,27.439000,5.236000,
  72          11.413000
  73          ],
  74         [18.999000,22.642000,8.158000,5.995000,5.495000,9.499000,8.092000,
  75          5.324000,4.468000,4.788000,8.248000,7.853000,27.533000,4.410000,
  76          4.622000,22.341000,22.313000,17.072000,26.731000,57.513000,33.001000,
  77          5.500000,28.297000,27.277000,26.462000,5.009000,27.361000,5.130000,
  78          10.955000
  79         ]
  80     ],
  81     [
  82         [18.238000,22.365000,8.555000,5.939000,5.437000,9.463000,7.047000,
  83          5.345000,4.517000,4.796000,8.593000,7.901000,27.499000,4.378000,
  84          5.040000,4.904000,4.816000,4.828000,4.853000,57.363000,34.184000,
  85          5.482000,28.190000,27.290000,26.694000,5.099000,4.905000,5.290000,
  86          4.813000
  87         ],
  88         [18.301000,22.522000,8.035000,6.021000,5.565000,9.037000,6.998000,
  89          5.321000,4.485000,4.768000,8.397000,7.865000,27.636000,4.640000,
  90          5.015000,4.962000,4.933000,4.977000,4.961000,60.648000,34.593000,
  91          5.538000,28.454000,27.297000,26.490000,5.099000,5,5.247000,4.945000
  92         ],
  93         [18.907000,23.368000,8.100000,6.169000,5.621000,9.971000,8.161000,
  94          5.331000,4.513000,4.837000,8.255000,7.852000,26.209000,4.388000,
  95          5.045000,5.029000,5.032000,4.946000,4.973000,60.334000,33.377000,
  96          5.499000,28.275000,27.550000,26.103000,5.108000,4.951000,5.285000,
  97          4.910000
  98         ],
  99        [18.715000,23.748000,8.128000,6.148000,5.691000,9.361000,8.106000,
 100         5.334000,4.528000,4.965000,8.261000,7.851000,27.282000,4.391000,
 101         4.949000,4.981000,4.964000,4.935000,4.933000,60.231000,33.361000,
 102         5.489000,28.106000,27.457000,26.648000,5.108000,4.963000,5.272000,
 103         4.954000
 104        ]
 105     ]
 106 ]
 107
 108 # Default options for the dry run
 109 DEFAULT_OPTIONS = {
 110     'debug_ignore_build': True,
 111     'debug_ignore_sync': True,
 112     'debug_ignore_perf_test': True,
 113     'debug_ignore_regression_confidence': True,
 114     'command': 'fake_command',
 115     'metric': 'fake/metric',
 116     'good_revision': 280000,
 117     'bad_revision': 280005,
 118 }
 119
 120 # This global is a placeholder for a generator to be defined by the testcases
 121 # that use _MockRunTest
 122 _MockResultsGenerator = (x for x in [])
 123
 124 def _FakeTestResult(values):
 125   result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
 126   success_code = 0
 127   return (result_dict, success_code)
 128
 129
 130 def _MockRunTests(*args, **kwargs):
 131   _, _ = args, kwargs
 132   return _FakeTestResult(_MockResultsGenerator.next())
 133
 134
 135 def _GetBisectPerformanceMetricsInstance(options_dict):
 136   """Returns an instance of the BisectPerformanceMetrics class."""
 137   opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
 138   return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 139
 140
 141 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True):
 142   """Returns the a copy of the default options dict plus some options."""
 143   result = dict(DEFAULT_OPTIONS)
 144   result.update({
 145       'improvement_direction': improvement_dir,
 146       'debug_fake_first_test_mean': fake_first,
 147       'debug_ignore_regression_confidence': ignore_confidence})
 148   return result
 149
 150
 151 def _GenericDryRun(options, print_results=False):
 152   """Performs a dry run of the bisector.
 153
 154   Args:
 155     options: Dictionary containing the options for the bisect instance.
 156     print_results: Boolean telling whether to call FormatAndPrintResults.
 157
 158   Returns:
 159     The results dictionary as returned by the bisect Run method.
 160   """
 161   # Disable rmtree to avoid deleting local trees.
 162   old_rmtree = shutil.rmtree
 163   try:
 164     shutil.rmtree = lambda path, onerror: None
 165     bisect_instance = _GetBisectPerformanceMetricsInstance(options)
 166     results = bisect_instance.Run(
 167         bisect_instance.opts.command, bisect_instance.opts.bad_revision,
 168         bisect_instance.opts.good_revision, bisect_instance.opts.metric)
 169
 170     if print_results:
 171       bisect_instance.printer.FormatAndPrintResults(results)
 172
 173     return results
 174   finally:
 175     shutil.rmtree = old_rmtree
 176
 177
 178 class BisectPerfRegressionTest(unittest.TestCase):
 179   """Test case for other functions and classes in bisect-perf-regression.py."""
 180
 181   def setUp(self):
 182     self.cwd = os.getcwd()
 183     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
 184                                           os.path.pardir, os.path.pardir)))
 185
 186   def tearDown(self):
 187     os.chdir(self.cwd)
 188
 189   def testParseDEPSStringManually(self):
 190     """Tests DEPS parsing."""
 191     deps_file_contents = """
 192     vars = {
 193         'ffmpeg_hash':
 194              '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 195         'webkit_url':
 196              'https://chromium.googlesource.com/chromium/blink.git',
 197         'git_url':
 198              'https://chromium.googlesource.com',
 199         'webkit_rev':
 200              '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 201         'angle_revision':
 202              '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 203     }"""
 204
 205     # Should only expect SVN/git revisions to come through, and URLs should be
 206     # filtered out.
 207     expected_vars_dict = {
 208         'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
 209         'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
 210         'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
 211     }
 212     # Testing private function.
 213     # pylint: disable=W0212
 214     vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
 215         deps_file_contents)
 216     self.assertEqual(vars_dict, expected_vars_dict)
 217
 218   def _AssertParseResult(self, expected_values, result_string):
 219     """Asserts some values are parsed from a RESULT line."""
 220     results_template = ('RESULT other_chart: other_trace= 123 count\n'
 221                         'RESULT my_chart: my_trace= %(value)s\n')
 222     results = results_template % {'value': result_string}
 223     metric = ['my_chart', 'my_trace']
 224     # Testing private function.
 225     # pylint: disable=W0212
 226     values = bisect_perf_regression._TryParseResultValuesFromOutput(
 227         metric, results)
 228     self.assertEqual(expected_values, values)
 229
 230   def testTryParseResultValuesFromOutput_WithSingleValue(self):
 231     """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
 232     self._AssertParseResult([66.88], '66.88 kb')
 233     self._AssertParseResult([66.88], '66.88 ')
 234     self._AssertParseResult([-66.88], '-66.88 kb')
 235     self._AssertParseResult([66], '66 kb')
 236     self._AssertParseResult([0.66], '.66 kb')
 237     self._AssertParseResult([], '. kb')
 238     self._AssertParseResult([], 'aaa kb')
 239
 240   def testTryParseResultValuesFromOutput_WithMultiValue(self):
 241     """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
 242     self._AssertParseResult([66.88], '[66.88] kb')
 243     self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
 244     self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
 245     self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
 246     self._AssertParseResult([-66, 99], '[-66,99] kb')
 247     self._AssertParseResult([-66, 99], '[-66,99,] kb')
 248     self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
 249     self._AssertParseResult([], '[] kb')
 250     self._AssertParseResult([], '[-66,abc] kb')
 251
 252   def testTryParseResultValuesFromOutputWithMeanStd(self):
 253     """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
 254     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 255     self._AssertParseResult([33.22], '{33.22, 3.6} kb')
 256     self._AssertParseResult([33.22], '{33.22,3.6}kb')
 257     self._AssertParseResult([33.22], '{33.22,3.6} kb')
 258     self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
 259     self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
 260     self._AssertParseResult([22], '{22,6}kb')
 261     self._AssertParseResult([.22], '{.22,6}kb')
 262     self._AssertParseResult([], '{.22,6, 44}kb')
 263     self._AssertParseResult([], '{}kb')
 264     self._AssertParseResult([], '{XYZ}kb')
 265
 266   def _AssertCompatibleCommand(
 267       self, expected_command, original_command, revision, target_platform):
 268     """Tests the modification of the command that might be done.
 269
 270     This modification to the command is done in order to get a Telemetry
 271     command that works; before some revisions, the browser name that Telemetry
 272     expects is different in some cases, but we want it to work anyway.
 273
 274     Specifically, only for android:
 275       After r276628, only android-chrome-shell works.
 276       Prior to r274857, only android-chromium-testshell works.
 277       In the range [274857, 276628], both work.
 278     """
 279     bisect_options = bisect_perf_regression.BisectOptions()
 280     bisect_options.output_buildbot_annotations = None
 281     bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
 282         bisect_options, os.getcwd())
 283     bisect_instance.opts.target_platform = target_platform
 284     git_revision = source_control.ResolveToRevision(
 285         revision, 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
 286     depot = 'chromium'
 287     command = bisect_instance.GetCompatibleCommand(
 288         original_command, git_revision, depot)
 289     self.assertEqual(expected_command, command)
 290
 291   def testGetCompatibleCommand_ChangeToTestShell(self):
 292     # For revisions <= r274857, only android-chromium-testshell is used.
 293     self._AssertCompatibleCommand(
 294         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 295         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 296         274857, 'android')
 297
 298   def testGetCompatibleCommand_ChangeToShell(self):
 299     # For revisions >= r276728, only android-chrome-shell can be used.
 300     self._AssertCompatibleCommand(
 301         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 302         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 303         276628, 'android')
 304
 305   def testGetCompatibleCommand_NoChange(self):
 306     # For revisions < r276728, android-chromium-testshell can be used.
 307     self._AssertCompatibleCommand(
 308         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 309         'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
 310         274858, 'android')
 311     # For revisions > r274857, android-chrome-shell can be used.
 312     self._AssertCompatibleCommand(
 313         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 314         'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
 315         274858, 'android')
 316
 317   def testGetCompatibleCommand_NonAndroidPlatform(self):
 318     # In most cases, there's no need to change Telemetry command.
 319     # For revisions >= r276728, only android-chrome-shell can be used.
 320     self._AssertCompatibleCommand(
 321         'tools/perf/run_benchmark -v --browser=release foo',
 322         'tools/perf/run_benchmark -v --browser=release foo',
 323         276628, 'chromium')
 324
 325   # This method doesn't reference self; it fails if an error is thrown.
 326   # pylint: disable=R0201
 327   def testDryRun(self):
 328     """Does a dry run of the bisect script.
 329
 330     This serves as a smoke test to catch errors in the basic execution of the
 331     script.
 332     """
 333     _GenericDryRun(DEFAULT_OPTIONS, True)
 334
 335   def testBisectImprovementDirectionFails(self):
 336     """Dry run of a bisect with an improvement instead of regression."""
 337     # Test result goes from 0 to 100 where higher is better
 338     results = _GenericDryRun(_GetExtendedOptions(1, 100))
 339     self.assertIsNotNone(results.error)
 340     self.assertIn('not a regression', results.error)
 341
 342     # Test result goes from 0 to -100 where lower is better
 343     results = _GenericDryRun(_GetExtendedOptions(-1, -100))
 344     self.assertIsNotNone(results.error)
 345     self.assertIn('not a regression', results.error)
 346
 347   def testBisectImprovementDirectionSucceeds(self):
 348     """Bisects with improvement direction matching regression range."""
 349     # Test result goes from 0 to 100 where lower is better
 350     results = _GenericDryRun(_GetExtendedOptions(-1, 100))
 351     self.assertIsNone(results.error)
 352     # Test result goes from 0 to -100 where higher is better
 353     results = _GenericDryRun(_GetExtendedOptions(1, -100))
 354     self.assertIsNone(results.error)
 355
 356   @mock.patch('bisect_perf_regression.BisectPerformanceMetrics.'
 357               'RunPerformanceTestAndParseResults', _MockRunTests)
 358   def testBisectStopsOnDoubtfulRegression(self):
 359     global _MockResultsGenerator
 360     _MockResultsGenerator = (rs for rs in CLEAR_NON_REGRESSION)
 361     results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 362     confidence_warnings = [x for x in results.warnings if x.startswith(
 363         '\nWe could not reproduce the regression')]
 364     self.assertGreater(len(confidence_warnings), 0)
 365
 366     _MockResultsGenerator = (rs for rs in ALMOST_REGRESSION)
 367     results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
 368     confidence_warnings = [x for x in results.warnings if x.startswith(
 369         '\nWe could not reproduce the regression')]
 370     self.assertGreater(len(confidence_warnings), 0)
 371
 372   @mock.patch('bisect_perf_regression.BisectPerformanceMetrics.'
 373               'RunPerformanceTestAndParseResults', _MockRunTests)
 374   def testBisectContinuesOnClearRegression(self):
 375     global _MockResultsGenerator
 376     _MockResultsGenerator = (rs for rs in CLEAR_REGRESSION)
 377     with self.assertRaises(StopIteration):
 378       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 379
 380     _MockResultsGenerator = (rs for rs in BARELY_REGRESSION)
 381     with self.assertRaises(StopIteration):
 382       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 383
 384     _MockResultsGenerator = (rs for rs in MULTIPLE_VALUES)
 385     with self.assertRaises(StopIteration):
 386       _GenericDryRun(_GetExtendedOptions(0, 0, False))
 387
 388   def testGetCommitPosition(self):
 389     cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
 390     self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
 391
 392     svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
 393     self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
 394
 395   def testGetCommitPositionForV8(self):
 396     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 397     v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
 398     depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
 399     self.assertEqual(
 400         23634, source_control.GetCommitPosition(v8_rev, depot_path))
 401
 402   def testGetCommitPositionForWebKit(self):
 403     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 404     wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
 405     depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
 406     self.assertEqual(
 407         181660, source_control.GetCommitPosition(wk_rev, depot_path))
 408
 409   def testUpdateDepsContent(self):
 410     bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
 411     deps_file = 'DEPS'
 412     # We are intentionally reading DEPS file contents instead of string literal
 413     # with few lines from DEPS because to check if the format we are expecting
 414     # to search is not changed in DEPS content.
 415     # TODO (prasadv): Add a separate test to validate the DEPS contents with the
 416     # format that bisect script expects.
 417     deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
 418     deps_key = 'v8_revision'
 419     depot = 'v8'
 420     git_revision = 'a12345789a23456789a123456789a123456789'
 421     updated_content = bisect_instance.UpdateDepsContents(
 422         deps_contents, depot, git_revision, deps_key)
 423     self.assertIsNotNone(updated_content)
 424     ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
 425     self.assertIsNotNone(re.search(ss, updated_content))
 426
 427
 428 class DepotDirectoryRegistryTest(unittest.TestCase):
 429
 430   def setUp(self):
 431     self.old_chdir = os.chdir
 432     os.chdir = self.mockChdir
 433     self.old_depot_names = bisect_utils.DEPOT_NAMES
 434     bisect_utils.DEPOT_NAMES = ['mock_depot']
 435     self.old_depot_deps_name = bisect_utils.DEPOT_DEPS_NAME
 436     bisect_utils.DEPOT_DEPS_NAME = {'mock_depot': {'src': 'src/foo'}}
 437
 438     self.registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
 439     self.cur_dir = None
 440
 441   def tearDown(self):
 442     os.chdir = self.old_chdir
 443     bisect_utils.DEPOT_NAMES = self.old_depot_names
 444     bisect_utils.DEPOT_DEPS_NAME = self.old_depot_deps_name
 445
 446   def mockChdir(self, new_dir):
 447     self.cur_dir = new_dir
 448
 449   def testReturnsCorrectResultForChrome(self):
 450     self.assertEqual(self.registry.GetDepotDir('chromium'), '/mock/src')
 451
 452   def testUsesDepotSpecToInitializeRegistry(self):
 453     self.assertEqual(self.registry.GetDepotDir('mock_depot'), '/mock/src/foo')
 454
 455   def testChangedTheDirectory(self):
 456     self.registry.ChangeToDepotDir('mock_depot')
 457     self.assertEqual(self.cur_dir, '/mock/src/foo')
 458
 459
 460 # The tests below test private functions (W0212).
 461 # pylint: disable=W0212
 462 class GitTryJobTestCases(unittest.TestCase):
 463
 464   """Test case for bisect try job."""
 465   def setUp(self):
 466     bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
 467     self.mock_bisect_utils = bisect_utils_patcher.start()
 468     self.addCleanup(bisect_utils_patcher.stop)
 469
 470   def _SetupRunGitMock(self, git_cmds):
 471     """Setup RunGit mock with expected output for given git command."""
 472     def side_effect(git_cmd_args):
 473       for val in git_cmds:
 474         if set(val[0]) == set(git_cmd_args):
 475           return val[1]
 476     self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
 477
 478   def _AssertRunGitExceptions(self, git_cmds, func, *args):
 479     """Setup RunGit mock and tests RunGitException.
 480
 481     Args:
 482       git_cmds: List of tuples with git command and expected output.
 483       func: Callback function to be executed.
 484       args: List of arguments to be passed to the function.
 485     """
 486     self._SetupRunGitMock(git_cmds)
 487     self.assertRaises(bisect_perf_regression.RunGitError,
 488                       func,
 489                       *args)
 490
 491   def testNotGitRepo(self):
 492     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 493     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 494     cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
 495     self._AssertRunGitExceptions(cmds,
 496                                  bisect_perf_regression._PrepareBisectBranch,
 497                                  parent_branch, new_branch)
 498
 499   def testFailedCheckoutMaster(self):
 500     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 501     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 502     cmds = [
 503         (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
 504         (['checkout', '-f', parent_branch], ('Checkout Failed', 1)),
 505     ]
 506     self._AssertRunGitExceptions(cmds,
 507                                  bisect_perf_regression._PrepareBisectBranch,
 508                                  parent_branch, new_branch)
 509
 510   def testDeleteBisectBranchIfExists(self):
 511     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 512     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 513     cmds = [
 514         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 515         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 516         (['branch', '-D', new_branch], ('Failed to delete branch', 128)),
 517     ]
 518     self._AssertRunGitExceptions(cmds,
 519                                  bisect_perf_regression._PrepareBisectBranch,
 520                                  parent_branch, new_branch)
 521
 522   def testCreatNewBranchFails(self):
 523     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 524     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 525     cmds = [
 526         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 527         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 528         (['branch', '-D', new_branch], ('None', 0)),
 529         (['update-index', '--refresh', '-q'], (None, 0)),
 530         (['diff-index', 'HEAD'], (None, 0)),
 531         (['checkout', '-b', new_branch], ('Failed to create branch', 128)),
 532     ]
 533     self._AssertRunGitExceptions(cmds,
 534                                  bisect_perf_regression._PrepareBisectBranch,
 535                                  parent_branch, new_branch)
 536
 537   def testSetUpstreamToFails(self):
 538     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 539     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 540     cmds = [
 541         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 542         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 543         (['branch', '-D', new_branch], ('None', 0)),
 544         (['update-index', '--refresh', '-q'], (None, 0)),
 545         (['diff-index', 'HEAD'], (None, 0)),
 546         (['checkout', '-b', new_branch], ('None', 0)),
 547         (['branch', '--set-upstream-to', parent_branch],
 548          ('Setuptream fails', 1)),
 549     ]
 550     self._AssertRunGitExceptions(cmds,
 551                                  bisect_perf_regression._PrepareBisectBranch,
 552                                  parent_branch, new_branch)
 553
 554   def testBuilderTryJobForException(self):
 555     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 556     bot_name = 'linux_perf_bisect_builder'
 557     bisect_job_name = 'testBisectJobname'
 558     patch = None
 559     patch_content = '/dev/null'
 560     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 561     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 562     try_cmd = [
 563         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 564         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 565         (['branch', '-D', new_branch], ('None', 0)),
 566         (['update-index', '--refresh', '-q'], (None, 0)),
 567         (['diff-index', 'HEAD'], (None, 0)),
 568         (['checkout', '-b', new_branch], ('None', 0)),
 569         (['branch', '--set-upstream-to', parent_branch],
 570          ('Setuptream fails', 0)),
 571         (['try',
 572           '-b', bot_name,
 573           '-r', git_revision,
 574           '-n', bisect_job_name,
 575           '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
 576           '--diff=%s' % patch_content
 577          ], (None, 1)),
 578     ]
 579     self._AssertRunGitExceptions(try_cmd,
 580                                  bisect_perf_regression._BuilderTryjob,
 581                                  git_revision, bot_name, bisect_job_name, patch)
 582
 583   def testBuilderTryJob(self):
 584     git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
 585     bot_name = 'linux_perf_bisect_builder'
 586     bisect_job_name = 'testBisectJobname'
 587     patch = None
 588     patch_content = '/dev/null'
 589     new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
 590     parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
 591     try_cmd = [
 592         (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
 593         (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
 594         (['branch', '-D', new_branch], ('None', 0)),
 595         (['update-index', '--refresh', '-q'], (None, 0)),
 596         (['diff-index', 'HEAD'], (None, 0)),
 597         (['checkout', '-b', new_branch], ('None', 0)),
 598         (['branch', '--set-upstream-to', parent_branch],
 599          ('Setuptream fails', 0)),
 600         (['try',
 601           '-b', bot_name,
 602           '-r', git_revision,
 603           '-n', bisect_job_name,
 604           '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
 605           '--diff=%s' % patch_content
 606          ], (None, 0)),
 607     ]
 608     self._SetupRunGitMock(try_cmd)
 609     bisect_perf_regression._BuilderTryjob(
 610         git_revision, bot_name, bisect_job_name, patch)
 611
 612
 613 if __name__ == '__main__':
 614   unittest.main()
 615