tools/run-bisect-perf-regression.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Run Performance Test Bisect Tool
   7
   8 This script is used by a try bot to run the bisect script with the parameters
   9 specified in the bisect config file. It checks out a copy of the depot in
  10 a subdirectory 'bisect' of the working directory provided, annd runs the
  11 bisect scrip there.
  12 """
  13
  14 import optparse
  15 import os
  16 import platform
  17 import subprocess
  18 import sys
  19 import traceback
  20
  21 from auto_bisect import bisect_perf_regression
  22 from auto_bisect import bisect_utils
  23 from auto_bisect import math_utils
  24
  25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
  26 CROS_IP_ENV = 'BISECT_CROS_IP'
  27
  28 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
  29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
  30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
  31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
  32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
  33     SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
  34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
  35
  36
  37 class Goma(object):
  38
  39   def __init__(self, path_to_goma):
  40     self._abs_path_to_goma = None
  41     self._abs_path_to_goma_file = None
  42     if not path_to_goma:
  43       return
  44     self._abs_path_to_goma = os.path.abspath(path_to_goma)
  45     filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
  46     self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
  47
  48   def __enter__(self):
  49     if self._HasGomaPath():
  50       self._SetupAndStart()
  51     return self
  52
  53   def __exit__(self, *_):
  54     if self._HasGomaPath():
  55       self._Stop()
  56
  57   def _HasGomaPath(self):
  58     return bool(self._abs_path_to_goma)
  59
  60   def _SetupEnvVars(self):
  61     if os.name == 'nt':
  62       os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  63           ' cl.exe')
  64       os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
  65           ' cl.exe')
  66     else:
  67       os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
  68           os.environ['PATH']])
  69
  70   def _SetupAndStart(self):
  71     """Sets up goma and launches it.
  72
  73     Args:
  74       path_to_goma: Path to goma directory.
  75
  76     Returns:
  77       True if successful."""
  78     self._SetupEnvVars()
  79
  80     # Sometimes goma is lingering around if something went bad on a previous
  81     # run. Stop it before starting a new process. Can ignore the return code
  82     # since it will return an error if it wasn't running.
  83     self._Stop()
  84
  85     if subprocess.call([self._abs_path_to_goma_file, 'start']):
  86       raise RuntimeError('Goma failed to start.')
  87
  88   def _Stop(self):
  89     subprocess.call([self._abs_path_to_goma_file, 'stop'])
  90
  91
  92 def _LoadConfigFile(config_file_path):
  93   """Attempts to load the specified config file as a module
  94   and grab the global config dict.
  95
  96   Args:
  97     config_file_path: Path to the config file.
  98
  99   Returns:
 100     If successful, returns the config dict loaded from the file. If no
 101     such dictionary could be loaded, returns the empty dictionary.
 102   """
 103   try:
 104     local_vars = {}
 105     execfile(config_file_path, local_vars)
 106     return local_vars['config']
 107   except Exception:
 108     print
 109     traceback.print_exc()
 110     print
 111     return {}
 112
 113
 114 def _ValidateConfigFile(config_contents, required_parameters):
 115   """Validates the config file contents, checking whether all values are
 116   non-empty.
 117
 118   Args:
 119     config_contents: A config dictionary.
 120     required_parameters: A list of parameters to check for.
 121
 122   Returns:
 123     True if valid.
 124   """
 125   for parameter in required_parameters:
 126     if parameter not in config_contents:
 127       return False
 128     value = config_contents[parameter]
 129     if not value or type(value) is not str:
 130       return False
 131   return True
 132
 133
 134 def _ValidatePerfConfigFile(config_contents):
 135   """Validates the perf config file contents.
 136
 137   This is used when we're doing a perf try job, rather than a bisect.
 138   The config file is called run-perf-test.cfg by default.
 139
 140   The parameters checked are the required parameters; any additional optional
 141   parameters won't be checked and validation will still pass.
 142
 143   Args:
 144     config_contents: A config dictionary.
 145
 146   Returns:
 147     True if valid.
 148   """
 149   required_parameters = [
 150       'command',
 151       'repeat_count',
 152       'truncate_percent',
 153       'max_time_minutes',
 154   ]
 155   return _ValidateConfigFile(config_contents, required_parameters)
 156
 157
 158 def _ValidateBisectConfigFile(config_contents):
 159   """Validates the bisect config file contents.
 160
 161   The parameters checked are the required parameters; any additional optional
 162   parameters won't be checked and validation will still pass.
 163
 164   Args:
 165     config_contents: A config dictionary.
 166
 167   Returns:
 168     True if valid.
 169   """
 170   required_params = [
 171       'command',
 172       'good_revision',
 173       'bad_revision',
 174       'metric',
 175       'repeat_count',
 176       'truncate_percent',
 177       'max_time_minutes',
 178   ]
 179   return _ValidateConfigFile(config_contents, required_params)
 180
 181
 182 def _OutputFailedResults(text_to_print):
 183   bisect_utils.OutputAnnotationStepStart('Results - Failed')
 184   print
 185   print text_to_print
 186   print
 187   bisect_utils.OutputAnnotationStepClosed()
 188
 189
 190 def _CreateBisectOptionsFromConfig(config):
 191   print config['command']
 192   opts_dict = {}
 193   opts_dict['command'] = config['command']
 194   opts_dict['metric'] = config.get('metric')
 195
 196   if config['repeat_count']:
 197     opts_dict['repeat_test_count'] = int(config['repeat_count'])
 198
 199   if config['truncate_percent']:
 200     opts_dict['truncate_percent'] = int(config['truncate_percent'])
 201
 202   if config['max_time_minutes']:
 203     opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
 204
 205   if config.has_key('use_goma'):
 206     opts_dict['use_goma'] = config['use_goma']
 207   if config.has_key('goma_dir'):
 208     opts_dict['goma_dir'] = config['goma_dir']
 209
 210   if config.has_key('improvement_direction'):
 211     opts_dict['improvement_direction'] = int(config['improvement_direction'])
 212
 213   if config.has_key('bug_id') and str(config['bug_id']).isdigit():
 214     opts_dict['bug_id'] = config['bug_id']
 215
 216   opts_dict['build_preference'] = 'ninja'
 217   opts_dict['output_buildbot_annotations'] = True
 218
 219   if '--browser=cros' in config['command']:
 220     opts_dict['target_platform'] = 'cros'
 221
 222     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 223       opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
 224       opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
 225     else:
 226       raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
 227           'BISECT_CROS_BOARD undefined.')
 228   elif 'android' in config['command']:
 229     if 'android-chrome-shell' in config['command']:
 230       opts_dict['target_platform'] = 'android'
 231     elif 'android-chrome' in config['command']:
 232       opts_dict['target_platform'] = 'android-chrome'
 233     else:
 234       opts_dict['target_platform'] = 'android'
 235
 236   return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
 237
 238
 239 def _RunPerformanceTest(config):
 240   """Runs a performance test with and without the current patch.
 241
 242   Args:
 243     config: Contents of the config file, a dictionary.
 244
 245   Attempts to build and run the current revision with and without the
 246   current patch, with the parameters passed in.
 247   """
 248   # Bisect script expects to be run from the src directory
 249   os.chdir(SRC_DIR)
 250
 251   bisect_utils.OutputAnnotationStepStart('Building With Patch')
 252
 253   opts = _CreateBisectOptionsFromConfig(config)
 254   b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
 255
 256   if bisect_utils.RunGClient(['runhooks']):
 257     raise RuntimeError('Failed to run gclient runhooks')
 258
 259   if not b.ObtainBuild('chromium'):
 260     raise RuntimeError('Patched version failed to build.')
 261
 262   bisect_utils.OutputAnnotationStepClosed()
 263   bisect_utils.OutputAnnotationStepStart('Running With Patch')
 264
 265   results_with_patch = b.RunPerformanceTestAndParseResults(
 266       opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
 267
 268   if results_with_patch[1]:
 269     raise RuntimeError('Patched version failed to run performance test.')
 270
 271   bisect_utils.OutputAnnotationStepClosed()
 272
 273   bisect_utils.OutputAnnotationStepStart('Reverting Patch')
 274   # TODO: When this is re-written to recipes, this should use bot_update's
 275   # revert mechanism to fully revert the client. But for now, since we know that
 276   # the perf try bot currently only supports src/ and src/third_party/WebKit, we
 277   # simply reset those two directories.
 278   bisect_utils.CheckRunGit(['reset', '--hard'])
 279   bisect_utils.CheckRunGit(['reset', '--hard'],
 280                            os.path.join('third_party', 'WebKit'))
 281   bisect_utils.OutputAnnotationStepClosed()
 282
 283   bisect_utils.OutputAnnotationStepStart('Building Without Patch')
 284
 285   if bisect_utils.RunGClient(['runhooks']):
 286     raise RuntimeError('Failed to run gclient runhooks')
 287
 288   if not b.ObtainBuild('chromium'):
 289     raise RuntimeError('Unpatched version failed to build.')
 290
 291   bisect_utils.OutputAnnotationStepClosed()
 292   bisect_utils.OutputAnnotationStepStart('Running Without Patch')
 293
 294   results_without_patch = b.RunPerformanceTestAndParseResults(
 295       opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
 296
 297   if results_without_patch[1]:
 298     raise RuntimeError('Unpatched version failed to run performance test.')
 299
 300   # Find the link to the cloud stored results file.
 301   output = results_without_patch[2]
 302   cloud_file_link = [t for t in output.splitlines()
 303       if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 304   if cloud_file_link:
 305     # What we're getting here is basically "View online at http://..." so parse
 306     # out just the URL portion.
 307     cloud_file_link = cloud_file_link[0]
 308     cloud_file_link = [t for t in cloud_file_link.split(' ')
 309         if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
 310     assert cloud_file_link, 'Couldn\'t parse URL from output.'
 311     cloud_file_link = cloud_file_link[0]
 312   else:
 313     cloud_file_link = ''
 314
 315   # Calculate the % difference in the means of the 2 runs.
 316   percent_diff_in_means = None
 317   std_err = None
 318   if (results_with_patch[0].has_key('mean') and
 319       results_with_patch[0].has_key('values')):
 320     percent_diff_in_means = (results_with_patch[0]['mean'] /
 321         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
 322     std_err = math_utils.PooledStandardError(
 323         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 324
 325   bisect_utils.OutputAnnotationStepClosed()
 326   if percent_diff_in_means is not None and std_err is not None:
 327     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
 328         (percent_diff_in_means, std_err))
 329     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
 330         'Std. Error'.center(20, ' '))
 331     print ' %s %s %s' % ('Patch'.center(10, ' '),
 332         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
 333         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
 334     print ' %s %s %s' % ('No Patch'.center(10, ' '),
 335         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
 336         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
 337     if cloud_file_link:
 338       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 339     bisect_utils.OutputAnnotationStepClosed()
 340   elif cloud_file_link:
 341     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 342
 343
 344 def _SetupAndRunPerformanceTest(config, path_to_goma):
 345   """Attempts to build and run the current revision with and without the
 346   current patch, with the parameters passed in.
 347
 348   Args:
 349     config: The config read from run-perf-test.cfg.
 350     path_to_goma: Path to goma directory.
 351
 352   Returns:
 353     An exit code: 0 on success, otherwise 1.
 354   """
 355   if platform.release() == 'XP':
 356     print 'Windows XP is not supported for perf try jobs because it lacks '
 357     print 'goma support. Please refer to crbug.com/330900.'
 358     return 1
 359   try:
 360     with Goma(path_to_goma) as _:
 361       config['use_goma'] = bool(path_to_goma)
 362       if config['use_goma']:
 363         config['goma_dir'] = os.path.abspath(path_to_goma)
 364       _RunPerformanceTest(config)
 365     return 0
 366   except RuntimeError, e:
 367     bisect_utils.OutputAnnotationStepClosed()
 368     _OutputFailedResults('Error: %s' % e.message)
 369     return 1
 370
 371
 372 def _RunBisectionScript(
 373     config, working_directory, path_to_goma, path_to_extra_src, dry_run):
 374   """Attempts to execute the bisect script with the given parameters.
 375
 376   Args:
 377     config: A dict containing the parameters to pass to the script.
 378     working_directory: A working directory to provide to the bisect script,
 379       where it will store it's own copy of the depot.
 380     path_to_goma: Path to goma directory.
 381     path_to_extra_src: Path to extra source file.
 382     dry_run: Do a dry run, skipping sync, build, and performance testing steps.
 383
 384   Returns:
 385     An exit status code: 0 on success, otherwise 1.
 386   """
 387   _PrintConfigStep(config)
 388
 389   cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
 390          '-c', config['command'],
 391          '-g', config['good_revision'],
 392          '-b', config['bad_revision'],
 393          '-m', config['metric'],
 394          '--working_directory', working_directory,
 395          '--output_buildbot_annotations']
 396
 397   if config.get('metric'):
 398     cmd.extend(['-m', config['metric']])
 399
 400   if config['repeat_count']:
 401     cmd.extend(['-r', config['repeat_count']])
 402
 403   if config['truncate_percent']:
 404     cmd.extend(['-t', config['truncate_percent']])
 405
 406   if config['max_time_minutes']:
 407     cmd.extend(['--max_time_minutes', config['max_time_minutes']])
 408
 409   if config.has_key('bisect_mode'):
 410     cmd.extend(['--bisect_mode', config['bisect_mode']])
 411
 412   if config.has_key('improvement_direction'):
 413     cmd.extend(['-d', config['improvement_direction']])
 414
 415   if config.has_key('bug_id'):
 416     cmd.extend(['--bug_id', config['bug_id']])
 417
 418   cmd.extend(['--build_preference', 'ninja'])
 419
 420   if '--browser=cros' in config['command']:
 421     cmd.extend(['--target_platform', 'cros'])
 422
 423     if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
 424       cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
 425       cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
 426     else:
 427       print ('Error: Cros build selected, but BISECT_CROS_IP or'
 428              'BISECT_CROS_BOARD undefined.\n')
 429       return 1
 430
 431   if 'android' in config['command']:
 432     if 'android-chrome-shell' in config['command']:
 433       cmd.extend(['--target_platform', 'android'])
 434     elif 'android-chrome' in config['command']:
 435       cmd.extend(['--target_platform', 'android-chrome'])
 436     else:
 437       cmd.extend(['--target_platform', 'android'])
 438
 439   if path_to_goma:
 440     # For Windows XP platforms, goma service is not supported.
 441     # Moreover we don't compile chrome when gs_bucket flag is set instead
 442     # use builds archives, therefore ignore goma service for Windows XP.
 443     # See http://crbug.com/330900.
 444     if config.get('gs_bucket') and platform.release() == 'XP':
 445       print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
 446              'on Windows XP platform. Please refer to crbug.com/330900.')
 447       path_to_goma = None
 448     cmd.append('--use_goma')
 449
 450   if path_to_extra_src:
 451     cmd.extend(['--extra_src', path_to_extra_src])
 452
 453   # These flags are used to download build archives from cloud storage if
 454   # available, otherwise will post a try_job_http request to build it on the
 455   # try server.
 456   if config.get('gs_bucket'):
 457     if config.get('builder_host') and config.get('builder_port'):
 458       cmd.extend(['--gs_bucket', config['gs_bucket'],
 459                   '--builder_host', config['builder_host'],
 460                   '--builder_port', config['builder_port']
 461                  ])
 462     else:
 463       print ('Error: Specified gs_bucket, but missing builder_host or '
 464              'builder_port information in config.')
 465       return 1
 466
 467   if dry_run:
 468     cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
 469         '--debug_ignore_perf_test'])
 470   cmd = [str(c) for c in cmd]
 471
 472   with Goma(path_to_goma) as _:
 473     return_code = subprocess.call(cmd)
 474
 475   if return_code:
 476     print ('Error: bisect_perf_regression.py returned with error %d\n'
 477            % return_code)
 478
 479   return return_code
 480
 481
 482 def _PrintConfigStep(config):
 483   """Prints out the given config, along with Buildbot annotations."""
 484   bisect_utils.OutputAnnotationStepStart('Config')
 485   print
 486   for k, v in config.iteritems():
 487     print '  %s : %s' % (k, v)
 488   print
 489   bisect_utils.OutputAnnotationStepClosed()
 490
 491
 492 def _OptionParser():
 493   """Returns the options parser for run-bisect-perf-regression.py."""
 494   usage = ('%prog [options] [-- chromium-options]\n'
 495            'Used by a try bot to run the bisection script using the parameters'
 496            ' provided in the auto_bisect/bisect.cfg file.')
 497   parser = optparse.OptionParser(usage=usage)
 498   parser.add_option('-w', '--working_directory',
 499                     type='str',
 500                     help='A working directory to supply to the bisection '
 501                     'script, which will use it as the location to checkout '
 502                     'a copy of the chromium depot.')
 503   parser.add_option('-p', '--path_to_goma',
 504                     type='str',
 505                     help='Path to goma directory. If this is supplied, goma '
 506                     'builds will be enabled.')
 507   parser.add_option('--path_to_config',
 508                     type='str',
 509                     help='Path to the config file to use. If this is supplied, '
 510                     'the bisect script will use this to override the default '
 511                     'config file path. The script will attempt to load it '
 512                     'as a bisect config first, then a perf config.')
 513   parser.add_option('--extra_src',
 514                     type='str',
 515                     help='Path to extra source file. If this is supplied, '
 516                     'bisect script will use this to override default behavior.')
 517   parser.add_option('--dry_run',
 518                     action="store_true",
 519                     help='The script will perform the full bisect, but '
 520                     'without syncing, building, or running the performance '
 521                     'tests.')
 522   return parser
 523
 524
 525 def main():
 526   """Entry point for run-bisect-perf-regression.py.
 527
 528   Reads the config file, and then tries to either bisect a regression or
 529   just run a performance test, depending on the particular config parameters
 530   specified in the config file.
 531   """
 532   parser = _OptionParser()
 533   opts, _ = parser.parse_args()
 534
 535   # Use the default config file path unless one was specified.
 536   config_path = BISECT_CONFIG_PATH
 537   if opts.path_to_config:
 538     config_path = opts.path_to_config
 539   config = _LoadConfigFile(config_path)
 540
 541   # Check if the config is valid for running bisect job.
 542   config_is_valid = _ValidateBisectConfigFile(config)
 543
 544   if config and config_is_valid:
 545     if not opts.working_directory:
 546       print 'Error: missing required parameter: --working_directory\n'
 547       parser.print_help()
 548       return 1
 549
 550     return _RunBisectionScript(
 551         config, opts.working_directory, opts.path_to_goma, opts.extra_src,
 552         opts.dry_run)
 553
 554   # If it wasn't valid for running a bisect, then maybe the user wanted
 555   # to run a perf test instead of a bisect job. Try reading any possible
 556   # perf test config files.
 557   perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
 558   for current_perf_cfg_file in perf_cfg_files:
 559     if opts.path_to_config:
 560       path_to_perf_cfg = opts.path_to_config
 561     else:
 562       path_to_perf_cfg = os.path.join(
 563           os.path.abspath(os.path.dirname(sys.argv[0])),
 564           current_perf_cfg_file)
 565
 566     config = _LoadConfigFile(path_to_perf_cfg)
 567     config_is_valid = _ValidatePerfConfigFile(config)
 568
 569     if config and config_is_valid:
 570       return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
 571
 572   print ('Error: Could not load config file. Double check your changes to '
 573          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
 574   return 1
 575
 576
 577 if __name__ == '__main__':
 578   sys.exit(main())