tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import platform
  19 import subprocess
  20 import sys
  21 import traceback
  22
  23 from auto_bisect import bisect_utils
  24
  25 bisect = imp.load_source('bisect-perf-regression',
  26     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  27         'bisect-perf-regression.py'))
  28
  29
  30 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  31 CROS_IP_ENV = 'BISECT_CROS_IP'
  32
  33
  34 class Goma(object):
  35
  36   def __init__(self, path_to_goma):
  37     self._abs_path_to_goma = None
  38     self._abs_path_to_goma_file = None
  39     if path_to_goma:
  40       self._abs_path_to_goma = os.path.abspath(path_to_goma)
  41       self._abs_path_to_goma_file = self._GetExecutablePath(
  42           self._abs_path_to_goma)
  43
  44   def __enter__(self):
  45     if self._HasGOMAPath():
  46       self._SetupAndStart()
  47     return self
  48
  49   def __exit__(self, *_):
  50     if self._HasGOMAPath():
  51       self._Stop()
  52
  53   def _HasGOMAPath(self):
  54     return bool(self._abs_path_to_goma)
  55
  56   def _GetExecutablePath(self, path_to_goma):
  57     if os.name == 'nt':
  58       return os.path.join(path_to_goma, 'goma_ctl.bat')
  59     else:
  60       return os.path.join(path_to_goma, 'goma_ctl.sh')
  61
  62   def _SetupEnvVars(self):
  63     if os.name == 'nt':
  64       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  67           ' cl.exe')
  68     else:
  69       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  70           os.environ['PATH']])
  71
  72   def _SetupAndStart(self):
  73     """Sets up GOMA and launches it.
  74
  75     Args:
  76       path_to_goma: Path to goma directory.
  77
  78     Returns:
  79       True if successful."""
  80     self._SetupEnvVars()
  81
  82     # Sometimes goma is lingering around if something went bad on a previous
  83     # run. Stop it before starting a new process. Can ignore the return code
  84     # since it will return an error if it wasn't running.
  85     self._Stop()
  86
  87     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  88       raise RuntimeError('GOMA failed to start.')
  89
  90   def _Stop(self):
  91     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  92
  93
  94
  95 def _LoadConfigFile(path_to_file):
  96   """Attempts to load the specified config file as a module
  97   and grab the global config dict.
  98
  99   Args:
 100     path_to_file: Path to the file.
 101
 102   Returns:
 103     The config dict which should be formatted as follows:
 104     {'command': string, 'good_revision': string, 'bad_revision': string
 105      'metric': string, etc...}.
 106     Returns None on failure.
 107   """
 108   try:
 109     local_vars = {}
 110     execfile(path_to_file, local_vars)
 111
 112     return local_vars['config']
 113   except:
 114     print
 115     traceback.print_exc()
 116     print
 117     return {}
 118
 119
 120 def _ValidateConfigFile(config_contents, valid_parameters):
 121   """Validates the config file contents, checking whether all values are
 122   non-empty.
 123
 124   Args:
 125     config_contents: Contents of the config file passed from _LoadConfigFile.
 126     valid_parameters: A list of parameters to check for.
 127
 128   Returns:
 129     True if valid.
 130   """
 131   try:
 132     [config_contents[current_parameter]
 133         for current_parameter in valid_parameters]
 134     config_has_values = [v and type(v) is str
 135         for v in config_contents.values() if v]
 136     return config_has_values
 137   except KeyError:
 138     return False
 139
 140
 141 def _ValidatePerfConfigFile(config_contents):
 142   """Validates that the perf config file contents. This is used when we're
 143   doing a perf try job, rather than a bisect. The file is called
 144   run-perf-test.cfg by default.
 145
 146   The parameters checked are the required parameters; any additional optional
 147   parameters won't be checked and validation will still pass.
 148
 149   Args:
 150     config_contents: Contents of the config file passed from _LoadConfigFile.
 151
 152   Returns:
 153     True if valid.
 154   """
 155   valid_parameters = ['command', 'metric', 'repeat_count', 'truncate_percent',
 156       'max_time_minutes']
 157   return _ValidateConfigFile(config_contents, valid_parameters)
 158
 159
 160 def _ValidateBisectConfigFile(config_contents):
 161   """Validates that the bisect config file contents. The parameters checked are
 162   the required parameters; any additional optional parameters won't be checked
 163   and validation will still pass.
 164
 165   Args:
 166     config_contents: Contents of the config file passed from _LoadConfigFile.
 167
 168   Returns:
 169     True if valid.
 170   """
 171   valid_params = ['command', 'good_revision', 'bad_revision', 'metric',
 172       'repeat_count', 'truncate_percent', 'max_time_minutes']
 173   return _ValidateConfigFile(config_contents, valid_params)
 174
 175
 176 def _OutputFailedResults(text_to_print):
 177   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 178   print
 179   print text_to_print
 180   print
 181   bisect_utils.OutputAnnotationStepClosed()
 182
 183
 184 def _CreateBisectOptionsFromConfig(config):
 185   opts_dict = {}
 186   opts_dict['command'] = config['command']
 187   opts_dict['metric'] = config['metric']
 188
 189   if config['repeat_count']:
 190     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 191
 192   if config['truncate_percent']:
 193     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 194
 195   if config['max_time_minutes']:
 196     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 197
 198   if config.has_key('use_goma'):
 199     opts_dict['use_goma'] = config['use_goma']
 200   if config.has_key('goma_dir'):
 201     opts_dict['goma_dir'] = config['goma_dir']
 202
 203   opts_dict['build_preference'] = 'ninja'
 204   opts_dict['output_buildbot_annotations'] = True
 205
 206   if '--browser=cros' in config['command']:
 207     opts_dict['target_platform'] = 'cros'
 208
 209     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 210       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 211       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 212     else:
 213       raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
 214           'BISECT_CROS_BOARD undefined.')
 215   elif 'android' in config['command']:
 216     if 'android-chrome' in config['command']:
 217       opts_dict['target_platform'] = 'android-chrome'
 218     else:
 219       opts_dict['target_platform'] = 'android'
 220
 221   return bisect.BisectOptions.FromDict(opts_dict)
 222
 223
 224 def _RunPerformanceTest(config, path_to_file):
 225   # Bisect script expects to be run from src
 226   os.chdir(os.path.join(path_to_file, '..'))
 227
 228   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 229
 230   opts = _CreateBisectOptionsFromConfig(config)
 231   b = bisect.BisectPerformanceMetrics(None, opts)
 232
 233   if bisect_utils.RunGClient(['runhooks']):
 234     raise RuntimeError('Failed to run gclient runhooks')
 235
 236   if not b.BuildCurrentRevision('chromium'):
 237     raise RuntimeError('Patched version failed to build.')
 238
 239   bisect_utils.OutputAnnotationStepClosed()
 240   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 241
 242   results_with_patch = b.RunPerformanceTestAndParseResults(
 243       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 244
 245   if results_with_patch[1]:
 246     raise RuntimeError('Patched version failed to run performance test.')
 247
 248   bisect_utils.OutputAnnotationStepClosed()
 249
 250   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 251   if bisect_utils.RunGClient(['revert']):
 252     raise RuntimeError('Failed to run gclient runhooks')
 253   bisect_utils.OutputAnnotationStepClosed()
 254
 255   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 256
 257   if bisect_utils.RunGClient(['runhooks']):
 258     raise RuntimeError('Failed to run gclient runhooks')
 259
 260   if not b.BuildCurrentRevision('chromium'):
 261     raise RuntimeError('Unpatched version failed to build.')
 262
 263   bisect_utils.OutputAnnotationStepClosed()
 264   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 265
 266   results_without_patch = b.RunPerformanceTestAndParseResults(
 267       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 268
 269   if results_without_patch[1]:
 270     raise RuntimeError('Unpatched version failed to run performance test.')
 271
 272   # Find the link to the cloud stored results file.
 273   output = results_without_patch[2]
 274   cloud_file_link = [t for t in output.splitlines()
 275       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 276   if cloud_file_link:
 277     # What we're getting here is basically "View online at http://..." so parse
 278     # out just the url portion.
 279     cloud_file_link = cloud_file_link[0]
 280     cloud_file_link = [t for t in cloud_file_link.split(' ')
 281         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 282     assert cloud_file_link, "Couldn't parse url from output."
 283     cloud_file_link = cloud_file_link[0]
 284   else:
 285     cloud_file_link = ''
 286
 287   # Calculate the % difference in the means of the 2 runs.
 288   percent_diff_in_means = (results_with_patch[0]['mean'] /
 289       max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 290   std_err = bisect.CalculatePooledStandardError(
 291       [results_with_patch[0]['values'], results_without_patch[0]['values']])
 292
 293   bisect_utils.OutputAnnotationStepClosed()
 294   bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 295       (percent_diff_in_means, std_err))
 296   print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 297       'Std. Error'.center(20, ' '))
 298   print ' %s %s %s' % ('Patch'.center(10, ' '),
 299       ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 300       ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 301   print ' %s %s %s' % ('No Patch'.center(10, ' '),
 302       ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 303       ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 304   if cloud_file_link:
 305     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 306   bisect_utils.OutputAnnotationStepClosed()
 307
 308
 309 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 310   """Attempts to build and run the current revision with and without the
 311   current patch, with the parameters passed in.
 312
 313   Args:
 314     config: The config read from run-perf-test.cfg.
 315     path_to_file: Path to the bisect-perf-regression.py script.
 316     path_to_goma: Path to goma directory.
 317
 318   Returns:
 319     0 on success, otherwise 1.
 320   """
 321   try:
 322     with Goma(path_to_goma) as goma:
 323       config['use_goma'] = bool(path_to_goma)
 324       config['goma_dir'] = os.path.abspath(path_to_goma)
 325       _RunPerformanceTest(config, path_to_file)
 326     return 0
 327   except RuntimeError, e:
 328     bisect_utils.OutputAnnotationStepClosed()
 329     _OutputFailedResults('Error: %s' % e.message)
 330     return 1
 331
 332
 333 def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
 334     path_to_extra_src, dry_run):
 335   """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
 336   passed in.
 337
 338   Args:
 339     config: A dict containing the parameters to pass to the script.
 340     working_directory: A working directory to provide to the
 341       bisect-perf-regression.py script, where it will store it's own copy of
 342       the depot.
 343     path_to_file: Path to the bisect-perf-regression.py script.
 344     path_to_goma: Path to goma directory.
 345     path_to_extra_src: Path to extra source file.
 346     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 347
 348   Returns:
 349     0 on success, otherwise 1.
 350   """
 351   bisect_utils.OutputAnnotationStepStart('Config')
 352   print
 353   for k, v in config.iteritems():
 354     print '  %s : %s' % (k, v)
 355   print
 356   bisect_utils.OutputAnnotationStepClosed()
 357
 358   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 359          '-c', config['command'],
 360          '-g', config['good_revision'],
 361          '-b', config['bad_revision'],
 362          '-m', config['metric'],
 363          '--working_directory', working_directory,
 364          '--output_buildbot_annotations']
 365
 366   if config['repeat_count']:
 367     cmd.extend(['-r', config['repeat_count']])
 368
 369   if config['truncate_percent']:
 370     cmd.extend(['-t', config['truncate_percent']])
 371
 372   if config['max_time_minutes']:
 373     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 374
 375   if config.has_key('bisect_mode'):
 376     cmd.extend(['--bisect_mode', config['bisect_mode']])
 377
 378   cmd.extend(['--build_preference', 'ninja'])
 379
 380   if '--browser=cros' in config['command']:
 381     cmd.extend(['--target_platform', 'cros'])
 382
 383     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 384       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 385       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 386     else:
 387       print 'Error: Cros build selected, but BISECT_CROS_IP or'\
 388             'BISECT_CROS_BOARD undefined.'
 389       print
 390       return 1
 391
 392   if 'android' in config['command']:
 393     if 'android-chrome' in config['command']:
 394       cmd.extend(['--target_platform', 'android-chrome'])
 395     else:
 396       cmd.extend(['--target_platform', 'android'])
 397
 398   if path_to_goma:
 399     # crbug.com/330900. For Windows XP platforms, GOMA service is not supported.
 400     # Moreover we don't compile chrome when gs_bucket flag is set instead
 401     # use builds archives, therefore ignore GOMA service for Windows XP.
 402     if config.get('gs_bucket') and platform.release() == 'XP':
 403       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 404              'on Windows XP platform. Please refer to crbug.com/330900.')
 405       path_to_goma = None
 406     cmd.append('--use_goma')
 407
 408   if path_to_extra_src:
 409     cmd.extend(['--extra_src', path_to_extra_src])
 410
 411   # These flags are used to download build archives from cloud storage if
 412   # available, otherwise will post a try_job_http request to build it on
 413   # tryserver.
 414   if config.get('gs_bucket'):
 415     if config.get('builder_host') and config.get('builder_port'):
 416       cmd.extend(['--gs_bucket', config['gs_bucket'],
 417                   '--builder_host', config['builder_host'],
 418                   '--builder_port', config['builder_port']
 419                  ])
 420     else:
 421       print ('Error: Specified gs_bucket, but missing builder_host or '
 422              'builder_port information in config.')
 423       return 1
 424
 425   if dry_run:
 426     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 427         '--debug_ignore_perf_test'])
 428   cmd = [str(c) for c in cmd]
 429
 430   with Goma(path_to_goma) as goma:
 431     return_code = subprocess.call(cmd)
 432
 433   if return_code:
 434     print 'Error: bisect-perf-regression.py returned with error %d' %\
 435         return_code
 436     print
 437
 438   return return_code
 439
 440
 441 def main():
 442
 443   usage = ('%prog [options] [-- chromium-options]\n'
 444            'Used by a trybot to run the bisection script using the parameters'
 445            ' provided in the run-bisect-perf-regression.cfg file.')
 446
 447   parser = optparse.OptionParser(usage=usage)
 448   parser.add_option('-w', '--working_directory',
 449                     type='str',
 450                     help='A working directory to supply to the bisection '
 451                     'script, which will use it as the location to checkout '
 452                     'a copy of the chromium depot.')
 453   parser.add_option('-p', '--path_to_goma',
 454                     type='str',
 455                     help='Path to goma directory. If this is supplied, goma '
 456                     'builds will be enabled.')
 457   parser.add_option('--path_to_config',
 458                     type='str',
 459                     help='Path to the config file to use. If this is supplied, '
 460                     'the bisect script will use this to override the default '
 461                     'config file path. The script will attempt to load it '
 462                     'as a bisect config first, then a perf config.')
 463   parser.add_option('--extra_src',
 464                     type='str',
 465                     help='Path to extra source file. If this is supplied, '
 466                     'bisect script will use this to override default behavior.')
 467   parser.add_option('--dry_run',
 468                     action="store_true",
 469                     help='The script will perform the full bisect, but '
 470                     'without syncing, building, or running the performance '
 471                     'tests.')
 472   (opts, args) = parser.parse_args()
 473
 474   path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
 475
 476   # If they've specified their own config file, use that instead.
 477   if opts.path_to_config:
 478     path_to_bisect_cfg = opts.path_to_config
 479   else:
 480     path_to_bisect_cfg = os.path.join(path_to_current_directory,
 481         'run-bisect-perf-regression.cfg')
 482
 483   config = _LoadConfigFile(path_to_bisect_cfg)
 484
 485   # Check if the config is valid.
 486   config_is_valid = _ValidateBisectConfigFile(config)
 487
 488   if config and config_is_valid:
 489     if not opts.working_directory:
 490       print 'Error: missing required parameter: --working_directory'
 491       print
 492       parser.print_help()
 493       return 1
 494
 495     return _RunBisectionScript(config, opts.working_directory,
 496         path_to_current_directory, opts.path_to_goma, opts.extra_src,
 497         opts.dry_run)
 498   else:
 499     perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
 500         'WebKit', 'Tools', 'run-perf-test.cfg')]
 501
 502     for current_perf_cfg_file in perf_cfg_files:
 503       if opts.path_to_config:
 504         path_to_perf_cfg = opts.path_to_config
 505       else:
 506         path_to_perf_cfg = os.path.join(
 507             os.path.abspath(os.path.dirname(sys.argv[0])),
 508             current_perf_cfg_file)
 509
 510       config = _LoadConfigFile(path_to_perf_cfg)
 511       config_is_valid = _ValidatePerfConfigFile(config)
 512
 513       if config and config_is_valid:
 514         return _SetupAndRunPerformanceTest(config, path_to_current_directory,
 515             opts.path_to_goma)
 516
 517     print 'Error: Could not load config file. Double check your changes to '\
 518           'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
 519     print
 520     return 1
 521
 522
 523 if __name__ == '__main__':
 524   sys.exit(main())