tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
   9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
  10 check out a copy of the depot in a subdirectory 'bisect' of the working
  11 directory provided, and run the bisect-perf-regression.py script there.
  12
  13 """
  14
  15 import imp
  16 import optparse
  17 import os
  18 import platform
  19 import subprocess
  20 import sys
  21 import traceback
  22
  23 from auto_bisect import bisect_utils
  24 from auto_bisect import math_utils
  25
  26 bisect = imp.load_source('bisect-perf-regression',
  27     os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
  28         'bisect-perf-regression.py'))
  29
  30
  31 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  32 CROS_IP_ENV = 'BISECT_CROS_IP'
  33
  34 # Default config file names.
  35 BISECT_REGRESSION_CONFIG = 'run-bisect-perf-regression.cfg'
  36 RUN_TEST_CONFIG = 'run-perf-test.cfg'
  37 WEBKIT_RUN_TEST_CONFIG = os.path.join(
  38     '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  39
  40 class Goma(object):
  41
  42   def __init__(self, path_to_goma):
  43     self._abs_path_to_goma = None
  44     self._abs_path_to_goma_file = None
  45     if not path_to_goma:
  46       return
  47     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  48     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  49     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  50
  51   def __enter__(self):
  52     if self._HasGomaPath():
  53       self._SetupAndStart()
  54     return self
  55
  56   def __exit__(self, *_):
  57     if self._HasGomaPath():
  58       self._Stop()
  59
  60   def _HasGomaPath(self):
  61     return bool(self._abs_path_to_goma)
  62
  63   def _SetupEnvVars(self):
  64     if os.name == 'nt':
  65       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  66           ' cl.exe')
  67       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  68           ' cl.exe')
  69     else:
  70       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  71           os.environ['PATH']])
  72
  73   def _SetupAndStart(self):
  74     """Sets up goma and launches it.
  75
  76     Args:
  77       path_to_goma: Path to goma directory.
  78
  79     Returns:
  80       True if successful."""
  81     self._SetupEnvVars()
  82
  83     # Sometimes goma is lingering around if something went bad on a previous
  84     # run. Stop it before starting a new process. Can ignore the return code
  85     # since it will return an error if it wasn't running.
  86     self._Stop()
  87
  88     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  89       raise RuntimeError('Goma failed to start.')
  90
  91   def _Stop(self):
  92     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  93
  94
  95 def _LoadConfigFile(config_file_path):
  96   """Attempts to load the specified config file as a module
  97   and grab the global config dict.
  98
  99   Args:
 100     config_file_path: Path to the config file.
 101
 102   Returns:
 103     If successful, returns the config dict loaded from the file. If no
 104     such dictionary could be loaded, returns the empty dictionary.
 105   """
 106   try:
 107     local_vars = {}
 108     execfile(config_file_path, local_vars)
 109     return local_vars['config']
 110   except Exception:
 111     print
 112     traceback.print_exc()
 113     print
 114     return {}
 115
 116
 117 def _ValidateConfigFile(config_contents, valid_parameters):
 118   """Validates the config file contents, checking whether all values are
 119   non-empty.
 120
 121   Args:
 122     config_contents: A config dictionary.
 123     valid_parameters: A list of parameters to check for.
 124
 125   Returns:
 126     True if valid.
 127   """
 128   for parameter in valid_parameters:
 129     if parameter not in config_contents:
 130       return False
 131     value = config_contents[parameter]
 132     if not value or type(value) is not str:
 133       return False
 134   return True
 135
 136
 137 def _ValidatePerfConfigFile(config_contents):
 138   """Validates the perf config file contents.
 139
 140   This is used when we're doing a perf try job, rather than a bisect.
 141   The config file is called run-perf-test.cfg by default.
 142
 143   The parameters checked are the required parameters; any additional optional
 144   parameters won't be checked and validation will still pass.
 145
 146   Args:
 147     config_contents: A config dictionary.
 148
 149   Returns:
 150     True if valid.
 151   """
 152   valid_parameters = [
 153       'command',
 154       'metric',
 155       'repeat_count',
 156       'truncate_percent',
 157       'max_time_minutes',
 158   ]
 159   return _ValidateConfigFile(config_contents, valid_parameters)
 160
 161
 162 def _ValidateBisectConfigFile(config_contents):
 163   """Validates the bisect config file contents.
 164
 165   The parameters checked are the required parameters; any additional optional
 166   parameters won't be checked and validation will still pass.
 167
 168   Args:
 169     config_contents: A config dictionary.
 170
 171   Returns:
 172     True if valid.
 173   """
 174   valid_params = [
 175       'command',
 176       'good_revision',
 177       'bad_revision',
 178       'metric',
 179       'repeat_count',
 180       'truncate_percent',
 181       'max_time_minutes',
 182   ]
 183   return _ValidateConfigFile(config_contents, valid_params)
 184
 185
 186 def _OutputFailedResults(text_to_print):
 187   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 188   print
 189   print text_to_print
 190   print
 191   bisect_utils.OutputAnnotationStepClosed()
 192
 193
 194 def _CreateBisectOptionsFromConfig(config):
 195   print config['command']
 196   opts_dict = {}
 197   opts_dict['command'] = config['command']
 198   opts_dict['metric'] = config['metric']
 199
 200   if config['repeat_count']:
 201     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 202
 203   if config['truncate_percent']:
 204     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 205
 206   if config['max_time_minutes']:
 207     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 208
 209   if config.has_key('use_goma'):
 210     opts_dict['use_goma'] = config['use_goma']
 211   if config.has_key('goma_dir'):
 212     opts_dict['goma_dir'] = config['goma_dir']
 213
 214   opts_dict['build_preference'] = 'ninja'
 215   opts_dict['output_buildbot_annotations'] = True
 216
 217   if '--browser=cros' in config['command']:
 218     opts_dict['target_platform'] = 'cros'
 219
 220     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 221       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 222       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 223     else:
 224       raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
 225           'BISECT_CROS_BOARD undefined.')
 226   elif 'android' in config['command']:
 227     if 'android-chrome-shell' in config['command']:
 228       opts_dict['target_platform'] = 'android'
 229     elif 'android-chrome' in config['command']:
 230       opts_dict['target_platform'] = 'android-chrome'
 231     else:
 232       opts_dict['target_platform'] = 'android'
 233
 234   return bisect.BisectOptions.FromDict(opts_dict)
 235
 236
 237 def _RunPerformanceTest(config, path_to_file):
 238   """Runs a performance test with and without the current patch.
 239
 240   Args:
 241     config: Contents of the config file, a dictionary.
 242     path_to_file: Path to the bisect-perf-regression.py script.
 243
 244   Attempts to build and run the current revision with and without the
 245   current patch, with the parameters passed in.
 246   """
 247   # Bisect script expects to be run from the src directory
 248   os.chdir(os.path.join(path_to_file, '..'))
 249
 250   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 251
 252   opts = _CreateBisectOptionsFromConfig(config)
 253   b = bisect.BisectPerformanceMetrics(None, opts)
 254
 255   if bisect_utils.RunGClient(['runhooks']):
 256     raise RuntimeError('Failed to run gclient runhooks')
 257
 258   if not b.BuildCurrentRevision('chromium'):
 259     raise RuntimeError('Patched version failed to build.')
 260
 261   bisect_utils.OutputAnnotationStepClosed()
 262   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 263
 264   results_with_patch = b.RunPerformanceTestAndParseResults(
 265       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 266
 267   if results_with_patch[1]:
 268     raise RuntimeError('Patched version failed to run performance test.')
 269
 270   bisect_utils.OutputAnnotationStepClosed()
 271
 272   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 273   if bisect_utils.RunGClient(['revert']):
 274     raise RuntimeError('Failed to run gclient runhooks')
 275   bisect_utils.OutputAnnotationStepClosed()
 276
 277   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 278
 279   if bisect_utils.RunGClient(['runhooks']):
 280     raise RuntimeError('Failed to run gclient runhooks')
 281
 282   if not b.BuildCurrentRevision('chromium'):
 283     raise RuntimeError('Unpatched version failed to build.')
 284
 285   bisect_utils.OutputAnnotationStepClosed()
 286   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 287
 288   results_without_patch = b.RunPerformanceTestAndParseResults(
 289       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 290
 291   if results_without_patch[1]:
 292     raise RuntimeError('Unpatched version failed to run performance test.')
 293
 294   # Find the link to the cloud stored results file.
 295   output = results_without_patch[2]
 296   cloud_file_link = [t for t in output.splitlines()
 297       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 298   if cloud_file_link:
 299     # What we're getting here is basically "View online at http://..." so parse
 300     # out just the url portion.
 301     cloud_file_link = cloud_file_link[0]
 302     cloud_file_link = [t for t in cloud_file_link.split(' ')
 303         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 304     assert cloud_file_link, "Couldn't parse url from output."
 305     cloud_file_link = cloud_file_link[0]
 306   else:
 307     cloud_file_link = ''
 308
 309   # Calculate the % difference in the means of the 2 runs.
 310   percent_diff_in_means = (results_with_patch[0]['mean'] /
 311       max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 312   std_err = math_utils.PooledStandardError(
 313       [results_with_patch[0]['values'], results_without_patch[0]['values']])
 314
 315   bisect_utils.OutputAnnotationStepClosed()
 316   bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 317       (percent_diff_in_means, std_err))
 318   print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 319       'Std. Error'.center(20, ' '))
 320   print ' %s %s %s' % ('Patch'.center(10, ' '),
 321       ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 322       ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 323   print ' %s %s %s' % ('No Patch'.center(10, ' '),
 324       ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 325       ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 326   if cloud_file_link:
 327     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 328   bisect_utils.OutputAnnotationStepClosed()
 329
 330
 331 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
 332   """Attempts to build and run the current revision with and without the
 333   current patch, with the parameters passed in.
 334
 335   Args:
 336     config: The config read from run-perf-test.cfg.
 337     path_to_file: Path to the bisect-perf-regression.py script.
 338     path_to_goma: Path to goma directory.
 339
 340   Returns:
 341     The exit code of bisect-perf-regression.py: 0 on success, otherwise 1.
 342   """
 343   try:
 344     with Goma(path_to_goma) as _:
 345       config['use_goma'] = bool(path_to_goma)
 346       config['goma_dir'] = os.path.abspath(path_to_goma)
 347       _RunPerformanceTest(config, path_to_file)
 348     return 0
 349   except RuntimeError, e:
 350     bisect_utils.OutputAnnotationStepClosed()
 351     _OutputFailedResults('Error: %s' % e.message)
 352     return 1
 353
 354
 355 def _RunBisectionScript(
 356     config, working_directory, path_to_file, path_to_goma, path_to_extra_src,
 357     dry_run):
 358   """Attempts to execute bisect-perf-regression.py with the given parameters.
 359
 360   Args:
 361     config: A dict containing the parameters to pass to the script.
 362     working_directory: A working directory to provide to the
 363       bisect-perf-regression.py script, where it will store it's own copy of
 364       the depot.
 365     path_to_file: Path to the bisect-perf-regression.py script.
 366     path_to_goma: Path to goma directory.
 367     path_to_extra_src: Path to extra source file.
 368     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 369
 370   Returns:
 371     An exit status code: 0 on success, otherwise 1.
 372   """
 373   _PrintConfigStep(config)
 374
 375   cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
 376          '-c', config['command'],
 377          '-g', config['good_revision'],
 378          '-b', config['bad_revision'],
 379          '-m', config['metric'],
 380          '--working_directory', working_directory,
 381          '--output_buildbot_annotations']
 382
 383   if config['repeat_count']:
 384     cmd.extend(['-r', config['repeat_count']])
 385
 386   if config['truncate_percent']:
 387     cmd.extend(['-t', config['truncate_percent']])
 388
 389   if config['max_time_minutes']:
 390     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 391
 392   if config.has_key('bisect_mode'):
 393     cmd.extend(['--bisect_mode', config['bisect_mode']])
 394
 395   cmd.extend(['--build_preference', 'ninja'])
 396
 397   if '--browser=cros' in config['command']:
 398     cmd.extend(['--target_platform', 'cros'])
 399
 400     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 401       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 402       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 403     else:
 404       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 405              'BISECT_CROS_BOARD undefined.\n')
 406       return 1
 407
 408   if 'android' in config['command']:
 409     if 'android-chrome-shell' in config['command']:
 410       cmd.extend(['--target_platform', 'android'])
 411     elif 'android-chrome' in config['command']:
 412       cmd.extend(['--target_platform', 'android-chrome'])
 413     else:
 414       cmd.extend(['--target_platform', 'android'])
 415
 416   if path_to_goma:
 417     # For Windows XP platforms, goma service is not supported.
 418     # Moreover we don't compile chrome when gs_bucket flag is set instead
 419     # use builds archives, therefore ignore goma service for Windows XP.
 420     # See http://crbug.com/330900.
 421     if config.get('gs_bucket') and platform.release() == 'XP':
 422       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 423              'on Windows XP platform. Please refer to crbug.com/330900.')
 424       path_to_goma = None
 425     cmd.append('--use_goma')
 426
 427   if path_to_extra_src:
 428     cmd.extend(['--extra_src', path_to_extra_src])
 429
 430   # These flags are used to download build archives from cloud storage if
 431   # available, otherwise will post a try_job_http request to build it on
 432   # tryserver.
 433   if config.get('gs_bucket'):
 434     if config.get('builder_host') and config.get('builder_port'):
 435       cmd.extend(['--gs_bucket', config['gs_bucket'],
 436                   '--builder_host', config['builder_host'],
 437                   '--builder_port', config['builder_port']
 438                  ])
 439     else:
 440       print ('Error: Specified gs_bucket, but missing builder_host or '
 441              'builder_port information in config.')
 442       return 1
 443
 444   if dry_run:
 445     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 446         '--debug_ignore_perf_test'])
 447   cmd = [str(c) for c in cmd]
 448
 449   with Goma(path_to_goma) as _:
 450     return_code = subprocess.call(cmd)
 451
 452   if return_code:
 453     print ('Error: bisect-perf-regression.py returned with error %d\n'
 454            % return_code)
 455
 456   return return_code
 457
 458
 459 def _PrintConfigStep(config):
 460   """Prints out the given config, along with Buildbot annotations."""
 461   bisect_utils.OutputAnnotationStepStart('Config')
 462   print
 463   for k, v in config.iteritems():
 464     print '  %s : %s' % (k, v)
 465   print
 466   bisect_utils.OutputAnnotationStepClosed()
 467
 468
 469 def _OptionParser():
 470   """Returns the options parser for run-bisect-perf-regression.py."""
 471   usage = ('%prog [options] [-- chromium-options]\n'
 472            'Used by a trybot to run the bisection script using the parameters'
 473            ' provided in the run-bisect-perf-regression.cfg file.')
 474   parser = optparse.OptionParser(usage=usage)
 475   parser.add_option('-w', '--working_directory',
 476                     type='str',
 477                     help='A working directory to supply to the bisection '
 478                     'script, which will use it as the location to checkout '
 479                     'a copy of the chromium depot.')
 480   parser.add_option('-p', '--path_to_goma',
 481                     type='str',
 482                     help='Path to goma directory. If this is supplied, goma '
 483                     'builds will be enabled.')
 484   parser.add_option('--path_to_config',
 485                     type='str',
 486                     help='Path to the config file to use. If this is supplied, '
 487                     'the bisect script will use this to override the default '
 488                     'config file path. The script will attempt to load it '
 489                     'as a bisect config first, then a perf config.')
 490   parser.add_option('--extra_src',
 491                     type='str',
 492                     help='Path to extra source file. If this is supplied, '
 493                     'bisect script will use this to override default behavior.')
 494   parser.add_option('--dry_run',
 495                     action="store_true",
 496                     help='The script will perform the full bisect, but '
 497                     'without syncing, building, or running the performance '
 498                     'tests.')
 499   return parser
 500
 501
 502 def main():
 503   """Entry point for run-bisect-perf-regression.py.
 504
 505   Reads the config file, and then tries to either bisect a regression or
 506   just run a performance test, depending on the particular config parameters
 507   specified in the config file.
 508   """
 509
 510   parser = _OptionParser()
 511   opts, _ = parser.parse_args()
 512
 513   current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
 514
 515   # Use the default config file path unless one was specified.
 516   config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG)
 517   if opts.path_to_config:
 518     config_path = opts.path_to_config
 519   config = _LoadConfigFile(config_path)
 520
 521   # Check if the config is valid for running bisect job.
 522   config_is_valid = _ValidateBisectConfigFile(config)
 523
 524   if config and config_is_valid:
 525     if not opts.working_directory:
 526       print 'Error: missing required parameter: --working_directory\n'
 527       parser.print_help()
 528       return 1
 529
 530     return _RunBisectionScript(
 531         config, opts.working_directory, current_dir,
 532         opts.path_to_goma, opts.extra_src, opts.dry_run)
 533
 534   # If it wasn't valid for running a bisect, then maybe the user wanted
 535   # to run a perf test instead of a bisect job. Try reading any possible
 536   # perf test config files.
 537   perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG]
 538   for current_perf_cfg_file in perf_cfg_files:
 539     if opts.path_to_config:
 540       path_to_perf_cfg = opts.path_to_config
 541     else:
 542       path_to_perf_cfg = os.path.join(
 543           os.path.abspath(os.path.dirname(sys.argv[0])),
 544           current_perf_cfg_file)
 545
 546     config = _LoadConfigFile(path_to_perf_cfg)
 547     config_is_valid = _ValidatePerfConfigFile(config)
 548
 549     if config and config_is_valid:
 550       return _SetupAndRunPerformanceTest(
 551           config, current_dir, opts.path_to_goma)
 552
 553   print ('Error: Could not load config file. Double check your changes to '
 554          'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax '
 555          'errors.\n')
 556   return 1
 557
 558
 559 if __name__ == '__main__':
 560   sys.exit(main())