tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import platform
  19 import subprocess
  20 import sys
  21 import traceback
  22
  23 import bisect_utils
  24 bisect = imp.load_source('bisect-perf-regression',
  25     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  26         'bisect-perf-regression.py'))
  27
  28
  29 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  30 CROS_IP_ENV = 'BISECT_CROS_IP'
  31
  32
  33 class Goma(object):
  34
  35   def __init__(self, path_to_goma):
  36     self._abs_path_to_goma = None
  37     self._abs_path_to_goma_file = None
  38     if path_to_goma:
  39       self._abs_path_to_goma = os.path.abspath(path_to_goma)
  40       self._abs_path_to_goma_file = self._GetExecutablePath(
  41           self._abs_path_to_goma)
  42
  43   def __enter__(self):
  44     if self._HasGOMAPath():
  45       self._SetupAndStart()
  46     return self
  47
  48   def __exit__(self, *_):
  49     if self._HasGOMAPath():
  50       self._Stop()
  51
  52   def _HasGOMAPath(self):
  53     return bool(self._abs_path_to_goma)
  54
  55   def _GetExecutablePath(self, path_to_goma):
  56     if os.name == 'nt':
  57       return os.path.join(path_to_goma, 'goma_ctl.bat')
  58     else:
  59       return os.path.join(path_to_goma, 'goma_ctl.sh')
  60
  61   def _SetupEnvVars(self):
  62     if os.name == 'nt':
  63       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  64           ' cl.exe')
  65       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  66           ' cl.exe')
  67     else:
  68       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  69           os.environ['PATH']])
  70
  71   def _SetupAndStart(self):
  72     """Sets up GOMA and launches it.
  73
  74     Args:
  75       path_to_goma: Path to goma directory.
  76
  77     Returns:
  78       True if successful."""
  79     self._SetupEnvVars()
  80
  81     # Sometimes goma is lingering around if something went bad on a previous
  82     # run. Stop it before starting a new process. Can ignore the return code
  83     # since it will return an error if it wasn't running.
  84     self._Stop()
  85
  86     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  87       raise RuntimeError('GOMA failed to start.')
  88
  89   def _Stop(self):
  90     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  91
  92
  93
  94 def _LoadConfigFile(path_to_file):
  95   """Attempts to load the specified config file as a module
  96   and grab the global config dict.
  97
  98   Args:
  99     path_to_file: Path to the file.
 100
 101   Returns:
 102     The config dict which should be formatted as follows:
 103     {'command': string, 'good_revision': string, 'bad_revision': string
 104      'metric': string, etc...}.
 105     Returns None on failure.
 106   """
 107   try:
 108     local_vars = {}
 109     execfile(path_to_file, local_vars)
 110
 111     return local_vars['config']
 112   except:
 113     print
 114     traceback.print_exc()
 115     print
 116     return {}
 117
 118
 119 def _ValidateConfigFile(config_contents, valid_parameters):
 120   """Validates the config file contents, checking whether all values are
 121   non-empty.
 122
 123   Args:
 124     config_contents: Contents of the config file passed from _LoadConfigFile.
 125     valid_parameters: A list of parameters to check for.
 126
 127   Returns:
 128     True if valid.
 129   """
 130   try:
 131     [config_contents[current_parameter]
 132         for current_parameter in valid_parameters]
 133     config_has_values = [v and type(v) is str
 134         for v in config_contents.values() if v]
 135     return config_has_values
 136   except KeyError:
 137     return False
 138
 139
 140 def _ValidatePerfConfigFile(config_contents):
 141   """Validates that the perf config file contents. This is used when we're
 142   doing a perf try job, rather than a bisect. The file is called
 143   run-perf-test.cfg by default.
 144
 145   The parameters checked are the required parameters; any additional optional
 146   parameters won't be checked and validation will still pass.
 147
 148   Args:
 149     config_contents: Contents of the config file passed from _LoadConfigFile.
 150
 151   Returns:
 152     True if valid.
 153   """
 154   valid_parameters = ['command', 'metric', 'repeat_count', 'truncate_percent',
 155       'max_time_minutes']
 156   return _ValidateConfigFile(config_contents, valid_parameters)
 157
 158
 159 def _ValidateBisectConfigFile(config_contents):
 160   """Validates that the bisect config file contents. The parameters checked are
 161   the required parameters; any additional optional parameters won't be checked
 162   and validation will still pass.
 163
 164   Args:
 165     config_contents: Contents of the config file passed from _LoadConfigFile.
 166
 167   Returns:
 168     True if valid.
 169   """
 170   valid_params = ['command', 'good_revision', 'bad_revision', 'metric',
 171       'repeat_count', 'truncate_percent', 'max_time_minutes']
 172   return _ValidateConfigFile(config_contents, valid_params)
 173
 174
 175 def _OutputFailedResults(text_to_print):
 176   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 177   print
 178   print text_to_print
 179   print
 180   bisect_utils.OutputAnnotationStepClosed()
 181
 182
 183 def _CreateBisectOptionsFromConfig(config):
 184   opts_dict = {}
 185   opts_dict['command'] = config['command']
 186   opts_dict['metric'] = config['metric']
 187
 188   if config['repeat_count']:
 189     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 190
 191   if config['truncate_percent']:
 192     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 193
 194   if config['max_time_minutes']:
 195     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 196
 197   if config.has_key('use_goma'):
 198     opts_dict['use_goma'] = config['use_goma']
 199   if config.has_key('goma_dir'):
 200     opts_dict['goma_dir'] = config['goma_dir']
 201
 202   opts_dict['build_preference'] = 'ninja'
 203   opts_dict['output_buildbot_annotations'] = True
 204
 205   if '--browser=cros' in config['command']:
 206     opts_dict['target_platform'] = 'cros'
 207
 208     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 209       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 210       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 211     else:
 212       raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
 213           'BISECT_CROS_BOARD undefined.')
 214   elif 'android' in config['command']:
 215     if 'android-chrome' in config['command']:
 216       opts_dict['target_platform'] = 'android-chrome'
 217     else:
 218       opts_dict['target_platform'] = 'android'
 219
 220   return bisect.BisectOptions.FromDict(opts_dict)
 221
 222
 223 def _RunPerformanceTest(config, path_to_file):
 224   # Bisect script expects to be run from src
 225   os.chdir(os.path.join(path_to_file, '..'))
 226
 227   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 228
 229   opts = _CreateBisectOptionsFromConfig(config)
 230   b = bisect.BisectPerformanceMetrics(None, opts)
 231
 232   if bisect_utils.RunGClient(['runhooks']):
 233     raise RuntimeError('Failed to run gclient runhooks')
 234
 235   if not b.BuildCurrentRevision('chromium'):
 236     raise RuntimeError('Patched version failed to build.')
 237
 238   bisect_utils.OutputAnnotationStepClosed()
 239   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 240
 241   results_with_patch = b.RunPerformanceTestAndParseResults(
 242       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 243
 244   if results_with_patch[1]:
 245     raise RuntimeError('Patched version failed to run performance test.')
 246
 247   bisect_utils.OutputAnnotationStepClosed()
 248
 249   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 250   if bisect_utils.RunGClient(['revert']):
 251     raise RuntimeError('Failed to run gclient runhooks')
 252   bisect_utils.OutputAnnotationStepClosed()
 253
 254   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 255
 256   if bisect_utils.RunGClient(['runhooks']):
 257     raise RuntimeError('Failed to run gclient runhooks')
 258
 259   if not b.BuildCurrentRevision('chromium'):
 260     raise RuntimeError('Unpatched version failed to build.')
 261
 262   bisect_utils.OutputAnnotationStepClosed()
 263   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 264
 265   results_without_patch = b.RunPerformanceTestAndParseResults(
 266       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 267
 268   if results_without_patch[1]:
 269     raise RuntimeError('Unpatched version failed to run performance test.')
 270
 271   # Find the link to the cloud stored results file.
 272   output = results_without_patch[2]
 273   cloud_file_link = [t for t in output.splitlines()
 274       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 275   if cloud_file_link:
 276     # What we're getting here is basically "View online at http://..." so parse
 277     # out just the url portion.
 278     cloud_file_link = cloud_file_link[0]
 279     cloud_file_link = [t for t in cloud_file_link.split(' ')
 280         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 281     assert cloud_file_link, "Couldn't parse url from output."
 282     cloud_file_link = cloud_file_link[0]
 283   else:
 284     cloud_file_link = ''
 285
 286   # Calculate the % difference in the means of the 2 runs.
 287   percent_diff_in_means = (results_with_patch[0]['mean'] /
 288       max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 289   std_err = bisect.CalculatePooledStandardError(
 290       [results_with_patch[0]['values'], results_without_patch[0]['values']])
 291
 292   bisect_utils.OutputAnnotationStepClosed()
 293   bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 294       (percent_diff_in_means, std_err))
 295   print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 296       'Std. Error'.center(20, ' '))
 297   print ' %s %s %s' % ('Patch'.center(10, ' '),
 298       ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 299       ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 300   print ' %s %s %s' % ('No Patch'.center(10, ' '),
 301       ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 302       ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 303   if cloud_file_link:
 304     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 305   bisect_utils.OutputAnnotationStepClosed()
 306
 307
 308 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 309   """Attempts to build and run the current revision with and without the
 310   current patch, with the parameters passed in.
 311
 312   Args:
 313     config: The config read from run-perf-test.cfg.
 314     path_to_file: Path to the bisect-perf-regression.py script.
 315     path_to_goma: Path to goma directory.
 316
 317   Returns:
 318     0 on success, otherwise 1.
 319   """
 320   try:
 321     with Goma(path_to_goma) as goma:
 322       config['use_goma'] = bool(path_to_goma)
 323       config['goma_dir'] = os.path.abspath(path_to_goma)
 324       _RunPerformanceTest(config, path_to_file)
 325     return 0
 326   except RuntimeError, e:
 327     bisect_utils.OutputAnnotationStepClosed()
 328     _OutputFailedResults('Error: %s' % e.message)
 329     return 1
 330
 331
 332 def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
 333     path_to_extra_src, dry_run):
 334   """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
 335   passed in.
 336
 337   Args:
 338     config: A dict containing the parameters to pass to the script.
 339     working_directory: A working directory to provide to the
 340       bisect-perf-regression.py script, where it will store it's own copy of
 341       the depot.
 342     path_to_file: Path to the bisect-perf-regression.py script.
 343     path_to_goma: Path to goma directory.
 344     path_to_extra_src: Path to extra source file.
 345     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 346
 347   Returns:
 348     0 on success, otherwise 1.
 349   """
 350   bisect_utils.OutputAnnotationStepStart('Config')
 351   print
 352   for k, v in config.iteritems():
 353     print '  %s : %s' % (k, v)
 354   print
 355   bisect_utils.OutputAnnotationStepClosed()
 356
 357   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 358          '-c', config['command'],
 359          '-g', config['good_revision'],
 360          '-b', config['bad_revision'],
 361          '-m', config['metric'],
 362          '--working_directory', working_directory,
 363          '--output_buildbot_annotations']
 364
 365   if config['repeat_count']:
 366     cmd.extend(['-r', config['repeat_count']])
 367
 368   if config['truncate_percent']:
 369     cmd.extend(['-t', config['truncate_percent']])
 370
 371   if config['max_time_minutes']:
 372     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 373
 374   if config.has_key('bisect_mode'):
 375     cmd.extend(['--bisect_mode', config['bisect_mode']])
 376
 377   cmd.extend(['--build_preference', 'ninja'])
 378
 379   if '--browser=cros' in config['command']:
 380     cmd.extend(['--target_platform', 'cros'])
 381
 382     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 383       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 384       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 385     else:
 386       print 'Error: Cros build selected, but BISECT_CROS_IP or'\
 387             'BISECT_CROS_BOARD undefined.'
 388       print
 389       return 1
 390
 391   if 'android' in config['command']:
 392     if 'android-chrome' in config['command']:
 393       cmd.extend(['--target_platform', 'android-chrome'])
 394     else:
 395       cmd.extend(['--target_platform', 'android'])
 396
 397   if path_to_goma:
 398     # crbug.com/330900. For Windows XP platforms, GOMA service is not supported.
 399     # Moreover we don't compile chrome when gs_bucket flag is set instead
 400     # use builds archives, therefore ignore GOMA service for Windows XP.
 401     if config.get('gs_bucket') and platform.release() == 'XP':
 402       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 403              'on Windows XP platform. Please refer to crbug.com/330900.')
 404       path_to_goma = None
 405     cmd.append('--use_goma')
 406
 407   if path_to_extra_src:
 408     cmd.extend(['--extra_src', path_to_extra_src])
 409
 410   # These flags are used to download build archives from cloud storage if
 411   # available, otherwise will post a try_job_http request to build it on
 412   # tryserver.
 413   if config.get('gs_bucket'):
 414     if config.get('builder_host') and config.get('builder_port'):
 415       cmd.extend(['--gs_bucket', config['gs_bucket'],
 416                   '--builder_host', config['builder_host'],
 417                   '--builder_port', config['builder_port']
 418                  ])
 419     else:
 420       print ('Error: Specified gs_bucket, but missing builder_host or '
 421              'builder_port information in config.')
 422       return 1
 423
 424   if dry_run:
 425     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 426         '--debug_ignore_perf_test'])
 427   cmd = [str(c) for c in cmd]
 428
 429   with Goma(path_to_goma) as goma:
 430     return_code = subprocess.call(cmd)
 431
 432   if return_code:
 433     print 'Error: bisect-perf-regression.py returned with error %d' %\
 434         return_code
 435     print
 436
 437   return return_code
 438
 439
 440 def main():
 441
 442   usage = ('%prog [options] [-- chromium-options]\n'
 443            'Used by a trybot to run the bisection script using the parameters'
 444            ' provided in the run-bisect-perf-regression.cfg file.')
 445
 446   parser = optparse.OptionParser(usage=usage)
 447   parser.add_option('-w', '--working_directory',
 448                     type='str',
 449                     help='A working directory to supply to the bisection '
 450                     'script, which will use it as the location to checkout '
 451                     'a copy of the chromium depot.')
 452   parser.add_option('-p', '--path_to_goma',
 453                     type='str',
 454                     help='Path to goma directory. If this is supplied, goma '
 455                     'builds will be enabled.')
 456   parser.add_option('--path_to_config',
 457                     type='str',
 458                     help='Path to the config file to use. If this is supplied, '
 459                     'the bisect script will use this to override the default '
 460                     'config file path. The script will attempt to load it '
 461                     'as a bisect config first, then a perf config.')
 462   parser.add_option('--extra_src',
 463                     type='str',
 464                     help='Path to extra source file. If this is supplied, '
 465                     'bisect script will use this to override default behavior.')
 466   parser.add_option('--dry_run',
 467                     action="store_true",
 468                     help='The script will perform the full bisect, but '
 469                     'without syncing, building, or running the performance '
 470                     'tests.')
 471   (opts, args) = parser.parse_args()
 472
 473   path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
 474
 475   # If they've specified their own config file, use that instead.
 476   if opts.path_to_config:
 477     path_to_bisect_cfg = opts.path_to_config
 478   else:
 479     path_to_bisect_cfg = os.path.join(path_to_current_directory,
 480         'run-bisect-perf-regression.cfg')
 481
 482   config = _LoadConfigFile(path_to_bisect_cfg)
 483
 484   # Check if the config is valid.
 485   config_is_valid = _ValidateBisectConfigFile(config)
 486
 487   if config and config_is_valid:
 488     if not opts.working_directory:
 489       print 'Error: missing required parameter: --working_directory'
 490       print
 491       parser.print_help()
 492       return 1
 493
 494     return _RunBisectionScript(config, opts.working_directory,
 495         path_to_current_directory, opts.path_to_goma, opts.extra_src,
 496         opts.dry_run)
 497   else:
 498     perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
 499         'WebKit', 'Tools', 'run-perf-test.cfg')]
 500
 501     for current_perf_cfg_file in perf_cfg_files:
 502       if opts.path_to_config:
 503         path_to_perf_cfg = opts.path_to_config
 504       else:
 505         path_to_perf_cfg = os.path.join(
 506             os.path.abspath(os.path.dirname(sys.argv[0])),
 507             current_perf_cfg_file)
 508
 509       config = _LoadConfigFile(path_to_perf_cfg)
 510       config_is_valid = _ValidatePerfConfigFile(config)
 511
 512       if config and config_is_valid:
 513         return _SetupAndRunPerformanceTest(config, path_to_current_directory,
 514             opts.path_to_goma)
 515
 516     print 'Error: Could not load config file. Double check your changes to '\
 517           'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
 518     print
 519     return 1
 520
 521
 522 if __name__ == '__main__':
 523   sys.exit(main())