tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import platform
  19 import subprocess
  20 import sys
  21 import traceback
  22
  23 import bisect_utils
  24 bisect = imp.load_source('bisect-perf-regression',
  25     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  26         'bisect-perf-regression.py'))
  27
  28
  29 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  30 CROS_IP_ENV = 'BISECT_CROS_IP'
  31
  32
  33 class Goma(object):
  34
  35   def __init__(self, path_to_goma):
  36     self._abs_path_to_goma = None
  37     self._abs_path_to_goma_file = None
  38     if path_to_goma:
  39       self._abs_path_to_goma = os.path.abspath(path_to_goma)
  40       self._abs_path_to_goma_file = self._GetExecutablePath(
  41           self._abs_path_to_goma)
  42
  43   def __enter__(self):
  44     if self._HasGOMAPath():
  45       self._SetupAndStart()
  46     return self
  47
  48   def __exit__(self, *_):
  49     if self._HasGOMAPath():
  50       self._Stop()
  51
  52   def _HasGOMAPath(self):
  53     return bool(self._abs_path_to_goma)
  54
  55   def _GetExecutablePath(self, path_to_goma):
  56     if os.name == 'nt':
  57       return os.path.join(path_to_goma, 'goma_ctl.bat')
  58     else:
  59       return os.path.join(path_to_goma, 'goma_ctl.sh')
  60
  61   def _SetupEnvVars(self):
  62     if os.name == 'nt':
  63       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  64           ' cl.exe')
  65       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  66           ' cl.exe')
  67     else:
  68       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  69           os.environ['PATH']])
  70
  71   def _SetupAndStart(self):
  72     """Sets up GOMA and launches it.
  73
  74     Args:
  75       path_to_goma: Path to goma directory.
  76
  77     Returns:
  78       True if successful."""
  79     self._SetupEnvVars()
  80
  81     # Sometimes goma is lingering around if something went bad on a previous
  82     # run. Stop it before starting a new process. Can ignore the return code
  83     # since it will return an error if it wasn't running.
  84     self._Stop()
  85
  86     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  87       raise RuntimeError('GOMA failed to start.')
  88
  89   def _Stop(self):
  90     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  91
  92
  93
  94 def _LoadConfigFile(path_to_file):
  95   """Attempts to load the specified config file as a module
  96   and grab the global config dict.
  97
  98   Args:
  99     path_to_file: Path to the file.
 100
 101   Returns:
 102     The config dict which should be formatted as follows:
 103     {'command': string, 'good_revision': string, 'bad_revision': string
 104      'metric': string, etc...}.
 105     Returns None on failure.
 106   """
 107   try:
 108     local_vars = {}
 109     execfile(path_to_file, local_vars)
 110
 111     return local_vars['config']
 112   except:
 113     print
 114     traceback.print_exc()
 115     print
 116     return {}
 117
 118
 119 def _ValidateConfigFile(config_contents, valid_parameters):
 120   """Validates the config file contents, checking whether all values are
 121   non-empty.
 122
 123   Args:
 124     config_contents: Contents of the config file passed from _LoadConfigFile.
 125     valid_parameters: A list of parameters to check for.
 126
 127   Returns:
 128     True if valid.
 129   """
 130   try:
 131     [config_contents[current_parameter]
 132         for current_parameter in valid_parameters]
 133     config_has_values = [v and type(v) is str
 134         for v in config_contents.values() if v]
 135     return config_has_values
 136   except KeyError:
 137     return False
 138
 139
 140 def _ValidatePerfConfigFile(config_contents):
 141   """Validates that the perf config file contents. This is used when we're
 142   doing a perf try job, rather than a bisect. The file is called
 143   run-perf-test.cfg by default.
 144
 145   The parameters checked are the required parameters; any additional optional
 146   parameters won't be checked and validation will still pass.
 147
 148   Args:
 149     config_contents: Contents of the config file passed from _LoadConfigFile.
 150
 151   Returns:
 152     True if valid.
 153   """
 154   valid_parameters = ['command', 'metric', 'repeat_count', 'truncate_percent',
 155       'max_time_minutes']
 156   return _ValidateConfigFile(config_contents, valid_parameters)
 157
 158
 159 def _ValidateBisectConfigFile(config_contents):
 160   """Validates that the bisect config file contents. The parameters checked are
 161   the required parameters; any additional optional parameters won't be checked
 162   and validation will still pass.
 163
 164   Args:
 165     config_contents: Contents of the config file passed from _LoadConfigFile.
 166
 167   Returns:
 168     True if valid.
 169   """
 170   valid_params = ['command', 'good_revision', 'bad_revision', 'metric',
 171       'repeat_count', 'truncate_percent', 'max_time_minutes']
 172   return _ValidateConfigFile(config_contents, valid_params)
 173
 174
 175 def _OutputFailedResults(text_to_print):
 176   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 177   print
 178   print text_to_print
 179   print
 180   bisect_utils.OutputAnnotationStepClosed()
 181
 182
 183 def _CreateBisectOptionsFromConfig(config):
 184   opts_dict = {}
 185   opts_dict['command'] = config['command']
 186   opts_dict['metric'] = config['metric']
 187
 188   if config['repeat_count']:
 189     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 190
 191   if config['truncate_percent']:
 192     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 193
 194   if config['max_time_minutes']:
 195     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 196
 197   if config.has_key('use_goma'):
 198     opts_dict['use_goma'] = config['use_goma']
 199
 200   opts_dict['build_preference'] = 'ninja'
 201   opts_dict['output_buildbot_annotations'] = True
 202
 203   if '--browser=cros' in config['command']:
 204     opts_dict['target_platform'] = 'cros'
 205
 206     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 207       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 208       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 209     else:
 210       raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
 211           'BISECT_CROS_BOARD undefined.')
 212   elif 'android' in config['command']:
 213     if 'android-chrome' in config['command']:
 214       opts_dict['target_platform'] = 'android-chrome'
 215     else:
 216       opts_dict['target_platform'] = 'android'
 217
 218   return bisect.BisectOptions.FromDict(opts_dict)
 219
 220
 221 def _RunPerformanceTest(config, path_to_file):
 222   # Bisect script expects to be run from src
 223   os.chdir(os.path.join(path_to_file, '..'))
 224
 225   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 226
 227   opts = _CreateBisectOptionsFromConfig(config)
 228   b = bisect.BisectPerformanceMetrics(None, opts)
 229
 230   if bisect_utils.RunGClient(['runhooks']):
 231     raise RuntimeError('Failed to run gclient runhooks')
 232
 233   if not b.BuildCurrentRevision('chromium'):
 234     raise RuntimeError('Patched version failed to build.')
 235
 236   bisect_utils.OutputAnnotationStepClosed()
 237   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 238
 239   results_with_patch = b.RunPerformanceTestAndParseResults(
 240       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 241
 242   if results_with_patch[1]:
 243     raise RuntimeError('Patched version failed to run performance test.')
 244
 245   bisect_utils.OutputAnnotationStepClosed()
 246
 247   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 248   if bisect_utils.RunGClient(['revert']):
 249     raise RuntimeError('Failed to run gclient runhooks')
 250   bisect_utils.OutputAnnotationStepClosed()
 251
 252   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 253
 254   if bisect_utils.RunGClient(['runhooks']):
 255     raise RuntimeError('Failed to run gclient runhooks')
 256
 257   if not b.BuildCurrentRevision('chromium'):
 258     raise RuntimeError('Unpatched version failed to build.')
 259
 260   bisect_utils.OutputAnnotationStepClosed()
 261   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 262
 263   results_without_patch = b.RunPerformanceTestAndParseResults(
 264       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 265
 266   if results_without_patch[1]:
 267     raise RuntimeError('Unpatched version failed to run performance test.')
 268
 269   # Find the link to the cloud stored results file.
 270   output = results_without_patch[2]
 271   cloud_file_link = [t for t in output.splitlines()
 272       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 273   if cloud_file_link:
 274     # What we're getting here is basically "View online at http://..." so parse
 275     # out just the url portion.
 276     cloud_file_link = cloud_file_link[0]
 277     cloud_file_link = [t for t in cloud_file_link.split(' ')
 278         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 279     assert cloud_file_link, "Couldn't parse url from output."
 280     cloud_file_link = cloud_file_link[0]
 281   else:
 282     cloud_file_link = ''
 283
 284   # Calculate the % difference in the means of the 2 runs.
 285   percent_diff_in_means = (results_with_patch[0]['mean'] /
 286       max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 287   std_err = bisect.CalculatePooledStandardError(
 288       [results_with_patch[0]['values'], results_without_patch[0]['values']])
 289
 290   bisect_utils.OutputAnnotationStepClosed()
 291   bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 292       (percent_diff_in_means, std_err))
 293   print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 294       'Std. Error'.center(20, ' '))
 295   print ' %s %s %s' % ('Patch'.center(10, ' '),
 296       ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 297       ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 298   print ' %s %s %s' % ('No Patch'.center(10, ' '),
 299       ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 300       ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 301   if cloud_file_link:
 302     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 303   bisect_utils.OutputAnnotationStepClosed()
 304
 305
 306 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 307   """Attempts to build and run the current revision with and without the
 308   current patch, with the parameters passed in.
 309
 310   Args:
 311     config: The config read from run-perf-test.cfg.
 312     path_to_file: Path to the bisect-perf-regression.py script.
 313     path_to_goma: Path to goma directory.
 314
 315   Returns:
 316     0 on success, otherwise 1.
 317   """
 318   try:
 319     with Goma(path_to_goma) as goma:
 320       config['use_goma'] = bool(path_to_goma)
 321       _RunPerformanceTest(config, path_to_file)
 322     return 0
 323   except RuntimeError, e:
 324     bisect_utils.OutputAnnotationStepClosed()
 325     _OutputFailedResults('Error: %s' % e.message)
 326     return 1
 327
 328
 329 def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
 330     path_to_extra_src, dry_run):
 331   """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
 332   passed in.
 333
 334   Args:
 335     config: A dict containing the parameters to pass to the script.
 336     working_directory: A working directory to provide to the
 337       bisect-perf-regression.py script, where it will store it's own copy of
 338       the depot.
 339     path_to_file: Path to the bisect-perf-regression.py script.
 340     path_to_goma: Path to goma directory.
 341     path_to_extra_src: Path to extra source file.
 342     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 343
 344   Returns:
 345     0 on success, otherwise 1.
 346   """
 347   bisect_utils.OutputAnnotationStepStart('Config')
 348   print
 349   for k, v in config.iteritems():
 350     print '  %s : %s' % (k, v)
 351   print
 352   bisect_utils.OutputAnnotationStepClosed()
 353
 354   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 355          '-c', config['command'],
 356          '-g', config['good_revision'],
 357          '-b', config['bad_revision'],
 358          '-m', config['metric'],
 359          '--working_directory', working_directory,
 360          '--output_buildbot_annotations']
 361
 362   if config['repeat_count']:
 363     cmd.extend(['-r', config['repeat_count']])
 364
 365   if config['truncate_percent']:
 366     cmd.extend(['-t', config['truncate_percent']])
 367
 368   if config['max_time_minutes']:
 369     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 370
 371   if config.has_key('bisect_mode'):
 372     cmd.extend(['--bisect_mode', config['bisect_mode']])
 373
 374   cmd.extend(['--build_preference', 'ninja'])
 375
 376   if '--browser=cros' in config['command']:
 377     cmd.extend(['--target_platform', 'cros'])
 378
 379     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 380       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 381       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 382     else:
 383       print 'Error: Cros build selected, but BISECT_CROS_IP or'\
 384             'BISECT_CROS_BOARD undefined.'
 385       print
 386       return 1
 387
 388   if 'android' in config['command']:
 389     if 'android-chrome' in config['command']:
 390       cmd.extend(['--target_platform', 'android-chrome'])
 391     else:
 392       cmd.extend(['--target_platform', 'android'])
 393
 394   if path_to_goma:
 395     # crbug.com/330900. For Windows XP platforms, GOMA service is not supported.
 396     # Moreover we don't compile chrome when gs_bucket flag is set instead
 397     # use builds archives, therefore ignore GOMA service for Windows XP.
 398     if config.get('gs_bucket') and platform.release() == 'XP':
 399       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 400              'on Windows XP platform. Please refer to crbug.com/330900.')
 401       path_to_goma = None
 402     cmd.append('--use_goma')
 403
 404   if path_to_extra_src:
 405     cmd.extend(['--extra_src', path_to_extra_src])
 406
 407   # These flags are used to download build archives from cloud storage if
 408   # available, otherwise will post a try_job_http request to build it on
 409   # tryserver.
 410   if config.get('gs_bucket'):
 411     if config.get('builder_host') and config.get('builder_port'):
 412       cmd.extend(['--gs_bucket', config['gs_bucket'],
 413                   '--builder_host', config['builder_host'],
 414                   '--builder_port', config['builder_port']
 415                  ])
 416     else:
 417       print ('Error: Specified gs_bucket, but missing builder_host or '
 418              'builder_port information in config.')
 419       return 1
 420
 421   if dry_run:
 422     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 423         '--debug_ignore_perf_test'])
 424   cmd = [str(c) for c in cmd]
 425
 426   with Goma(path_to_goma) as goma:
 427     return_code = subprocess.call(cmd)
 428
 429   if return_code:
 430     print 'Error: bisect-perf-regression.py returned with error %d' %\
 431         return_code
 432     print
 433
 434   return return_code
 435
 436
 437 def main():
 438
 439   usage = ('%prog [options] [-- chromium-options]\n'
 440            'Used by a trybot to run the bisection script using the parameters'
 441            ' provided in the run-bisect-perf-regression.cfg file.')
 442
 443   parser = optparse.OptionParser(usage=usage)
 444   parser.add_option('-w', '--working_directory',
 445                     type='str',
 446                     help='A working directory to supply to the bisection '
 447                     'script, which will use it as the location to checkout '
 448                     'a copy of the chromium depot.')
 449   parser.add_option('-p', '--path_to_goma',
 450                     type='str',
 451                     help='Path to goma directory. If this is supplied, goma '
 452                     'builds will be enabled.')
 453   parser.add_option('--path_to_config',
 454                     type='str',
 455                     help='Path to the config file to use. If this is supplied, '
 456                     'the bisect script will use this to override the default '
 457                     'config file path. The script will attempt to load it '
 458                     'as a bisect config first, then a perf config.')
 459   parser.add_option('--extra_src',
 460                     type='str',
 461                     help='Path to extra source file. If this is supplied, '
 462                     'bisect script will use this to override default behavior.')
 463   parser.add_option('--dry_run',
 464                     action="store_true",
 465                     help='The script will perform the full bisect, but '
 466                     'without syncing, building, or running the performance '
 467                     'tests.')
 468   (opts, args) = parser.parse_args()
 469
 470   path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
 471
 472   # If they've specified their own config file, use that instead.
 473   if opts.path_to_config:
 474     path_to_bisect_cfg = opts.path_to_config
 475   else:
 476     path_to_bisect_cfg = os.path.join(path_to_current_directory,
 477         'run-bisect-perf-regression.cfg')
 478
 479   config = _LoadConfigFile(path_to_bisect_cfg)
 480
 481   # Check if the config is valid.
 482   config_is_valid = _ValidateBisectConfigFile(config)
 483
 484   if config and config_is_valid:
 485     if not opts.working_directory:
 486       print 'Error: missing required parameter: --working_directory'
 487       print
 488       parser.print_help()
 489       return 1
 490
 491     return _RunBisectionScript(config, opts.working_directory,
 492         path_to_current_directory, opts.path_to_goma, opts.extra_src,
 493         opts.dry_run)
 494   else:
 495     perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
 496         'WebKit', 'Tools', 'run-perf-test.cfg')]
 497
 498     for current_perf_cfg_file in perf_cfg_files:
 499       if opts.path_to_config:
 500         path_to_perf_cfg = opts.path_to_config
 501       else:
 502         path_to_perf_cfg = os.path.join(
 503             os.path.abspath(os.path.dirname(sys.argv[0])),
 504             current_perf_cfg_file)
 505
 506       config = _LoadConfigFile(path_to_perf_cfg)
 507       config_is_valid = _ValidatePerfConfigFile(config)
 508
 509       if config and config_is_valid:
 510         return _SetupAndRunPerformanceTest(config, path_to_current_directory,
 511             opts.path_to_goma)
 512
 513     print 'Error: Could not load config file. Double check your changes to '\
 514           'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
 515     print
 516     return 1
 517
 518
 519 if __name__ == '__main__':
 520   sys.exit(main())