2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Run Performance Test Bisect Tool
8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
10 check out a copy of the depot in a subdirectory 'bisect' of the working
11 directory provided, and run the bisect-perf-regression.py script there.
23 bisect
= imp
.load_source('bisect-perf-regression',
24 os
.path
.join(os
.path
.abspath(os
.path
.dirname(sys
.argv
[0])),
25 'bisect-perf-regression.py'))
28 CROS_BOARD_ENV
= 'BISECT_CROS_BOARD'
29 CROS_IP_ENV
= 'BISECT_CROS_IP'
34 def __init__(self
, path_to_goma
):
35 self
._abs
_path
_to
_goma
= None
36 self
._abs
_path
_to
_goma
_file
= None
38 self
._abs
_path
_to
_goma
= os
.path
.abspath(path_to_goma
)
39 self
._abs
_path
_to
_goma
_file
= self
._GetExecutablePath
(
40 self
._abs
_path
_to
_goma
)
43 if self
._HasGOMAPath
():
47 def __exit__(self
, *_
):
48 if self
._HasGOMAPath
():
51 def _HasGOMAPath(self
):
52 return bool(self
._abs
_path
_to
_goma
)
54 def _GetExecutablePath(self
, path_to_goma
):
56 return os
.path
.join(path_to_goma
, 'goma_ctl.bat')
58 return os
.path
.join(path_to_goma
, 'goma_ctl.sh')
60 def _SetupEnvVars(self
):
62 os
.environ
['CC'] = (os
.path
.join(self
._abs
_path
_to
_goma
, 'gomacc.exe') +
64 os
.environ
['CXX'] = (os
.path
.join(self
._abs
_path
_to
_goma
, 'gomacc.exe') +
67 os
.environ
['PATH'] = os
.pathsep
.join([self
._abs
_path
_to
_goma
,
70 def _SetupAndStart(self
):
71 """Sets up GOMA and launches it.
74 path_to_goma: Path to goma directory.
77 True if successful."""
80 # Sometimes goma is lingering around if something went bad on a previous
81 # run. Stop it before starting a new process. Can ignore the return code
82 # since it will return an error if it wasn't running.
85 if subprocess
.call([self
._abs
_path
_to
_goma
_file
, 'start']):
86 raise RuntimeError('GOMA failed to start.')
89 subprocess
.call([self
._abs
_path
_to
_goma
_file
, 'stop'])
93 def _LoadConfigFile(path_to_file
):
94 """Attempts to load the specified config file as a module
95 and grab the global config dict.
98 path_to_file: Path to the file.
101 The config dict which should be formatted as follows:
102 {'command': string, 'good_revision': string, 'bad_revision': string
103 'metric': string, etc...}.
104 Returns None on failure.
108 execfile(path_to_file
, local_vars
)
110 return local_vars
['config']
113 traceback
.print_exc()
118 def _OutputFailedResults(text_to_print
):
119 bisect_utils
.OutputAnnotationStepStart('Results - Failed')
123 bisect_utils
.OutputAnnotationStepClosed()
126 def _CreateBisectOptionsFromConfig(config
):
128 opts_dict
['command'] = config
['command']
129 opts_dict
['metric'] = config
['metric']
131 if config
['repeat_count']:
132 opts_dict
['repeat_test_count'] = int(config
['repeat_count'])
134 if config
['truncate_percent']:
135 opts_dict
['truncate_percent'] = int(config
['truncate_percent'])
137 if config
['max_time_minutes']:
138 opts_dict
['max_time_minutes'] = int(config
['max_time_minutes'])
140 if config
.has_key('use_goma'):
141 opts_dict
['use_goma'] = config
['use_goma']
143 opts_dict
['build_preference'] = 'ninja'
144 opts_dict
['output_buildbot_annotations'] = True
146 if '--browser=cros' in config
['command']:
147 opts_dict
['target_platform'] = 'cros'
149 if os
.environ
[CROS_BOARD_ENV
] and os
.environ
[CROS_IP_ENV
]:
150 opts_dict
['cros_board'] = os
.environ
[CROS_BOARD_ENV
]
151 opts_dict
['cros_remote_ip'] = os
.environ
[CROS_IP_ENV
]
153 raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
154 'BISECT_CROS_BOARD undefined.')
155 elif 'android' in config
['command']:
156 if 'android-chrome' in config
['command']:
157 opts_dict
['target_platform'] = 'android-chrome'
159 opts_dict
['target_platform'] = 'android'
161 return bisect
.BisectOptions
.FromDict(opts_dict
)
164 def _RunPerformanceTest(config
, path_to_file
):
165 # Bisect script expects to be run from src
166 os
.chdir(os
.path
.join(path_to_file
, '..'))
168 bisect_utils
.OutputAnnotationStepStart('Building With Patch')
170 opts
= _CreateBisectOptionsFromConfig(config
)
171 b
= bisect
.BisectPerformanceMetrics(None, opts
)
173 if bisect_utils
.RunGClient(['runhooks']):
174 raise RuntimeError('Failed to run gclient runhooks')
176 if not b
.BuildCurrentRevision('chromium'):
177 raise RuntimeError('Patched version failed to build.')
179 bisect_utils
.OutputAnnotationStepClosed()
180 bisect_utils
.OutputAnnotationStepStart('Running With Patch')
182 results_with_patch
= b
.RunPerformanceTestAndParseResults(
183 opts
.command
, opts
.metric
, reset_on_first_run
=True, results_label
='Patch')
185 if results_with_patch
[1]:
186 raise RuntimeError('Patched version failed to run performance test.')
188 bisect_utils
.OutputAnnotationStepClosed()
190 bisect_utils
.OutputAnnotationStepStart('Reverting Patch')
191 if bisect_utils
.RunGClient(['revert']):
192 raise RuntimeError('Failed to run gclient runhooks')
193 bisect_utils
.OutputAnnotationStepClosed()
195 bisect_utils
.OutputAnnotationStepStart('Building Without Patch')
197 if bisect_utils
.RunGClient(['runhooks']):
198 raise RuntimeError('Failed to run gclient runhooks')
200 if not b
.BuildCurrentRevision('chromium'):
201 raise RuntimeError('Unpatched version failed to build.')
203 bisect_utils
.OutputAnnotationStepClosed()
204 bisect_utils
.OutputAnnotationStepStart('Running Without Patch')
206 results_without_patch
= b
.RunPerformanceTestAndParseResults(
207 opts
.command
, opts
.metric
, upload_on_last_run
=True, results_label
='ToT')
209 if results_without_patch
[1]:
210 raise RuntimeError('Unpatched version failed to run performance test.')
212 # Find the link to the cloud stored results file.
213 output
= results_without_patch
[2]
214 cloud_file_link
= [t
for t
in output
.splitlines()
215 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t
]
217 # What we're getting here is basically "View online at http://..." so parse
218 # out just the url portion.
219 cloud_file_link
= cloud_file_link
[0]
220 cloud_file_link
= [t
for t
in cloud_file_link
.split(' ')
221 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t
]
222 assert cloud_file_link
, "Couldn't parse url from output."
223 cloud_file_link
= cloud_file_link
[0]
227 # Calculate the % difference in the means of the 2 runs.
228 percent_diff_in_means
= (results_with_patch
[0]['mean'] /
229 max(0.0001, results_without_patch
[0]['mean'])) * 100.0 - 100.0
230 std_err
= bisect
.CalculatePooledStandardError(
231 [results_with_patch
[0]['values'], results_without_patch
[0]['values']])
233 bisect_utils
.OutputAnnotationStepClosed()
234 bisect_utils
.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
235 (percent_diff_in_means
, std_err
))
236 print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
237 'Std. Error'.center(20, ' '))
238 print ' %s %s %s' % ('Patch'.center(10, ' '),
239 ('%.02f' % results_with_patch
[0]['mean']).center(20, ' '),
240 ('%.02f' % results_with_patch
[0]['std_err']).center(20, ' '))
241 print ' %s %s %s' % ('No Patch'.center(10, ' '),
242 ('%.02f' % results_without_patch
[0]['mean']).center(20, ' '),
243 ('%.02f' % results_without_patch
[0]['std_err']).center(20, ' '))
245 bisect_utils
.OutputAnnotationStepLink('HTML Results', cloud_file_link
)
246 bisect_utils
.OutputAnnotationStepClosed()
249 def _SetupAndRunPerformanceTest(config
, path_to_file
, path_to_goma
):
250 """Attempts to build and run the current revision with and without the
251 current patch, with the parameters passed in.
254 config: The config read from run-perf-test.cfg.
255 path_to_file: Path to the bisect-perf-regression.py script.
256 path_to_goma: Path to goma directory.
259 0 on success, otherwise 1.
262 with
Goma(path_to_goma
) as goma
:
263 config
['use_goma'] = bool(path_to_goma
)
264 _RunPerformanceTest(config
, path_to_file
)
266 except RuntimeError, e
:
267 bisect_utils
.OutputAnnotationStepClosed()
268 _OutputFailedResults('Error: %s' % e
.message
)
272 def _RunBisectionScript(config
, working_directory
, path_to_file
, path_to_goma
,
273 path_to_extra_src
, dry_run
):
274 """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
278 config: A dict containing the parameters to pass to the script.
279 working_directory: A working directory to provide to the
280 bisect-perf-regression.py script, where it will store it's own copy of
282 path_to_file: Path to the bisect-perf-regression.py script.
283 path_to_goma: Path to goma directory.
284 path_to_extra_src: Path to extra source file.
285 dry_run: Do a dry run, skipping sync, build, and performance testing steps.
288 0 on success, otherwise 1.
290 bisect_utils
.OutputAnnotationStepStart('Config')
292 for k
, v
in config
.iteritems():
293 print ' %s : %s' % (k
, v
)
295 bisect_utils
.OutputAnnotationStepClosed()
297 cmd
= ['python', os
.path
.join(path_to_file
, 'bisect-perf-regression.py'),
298 '-c', config
['command'],
299 '-g', config
['good_revision'],
300 '-b', config
['bad_revision'],
301 '-m', config
['metric'],
302 '--working_directory', working_directory
,
303 '--output_buildbot_annotations']
305 if config
['repeat_count']:
306 cmd
.extend(['-r', config
['repeat_count']])
308 if config
['truncate_percent']:
309 cmd
.extend(['-t', config
['truncate_percent']])
311 if config
['max_time_minutes']:
312 cmd
.extend(['--max_time_minutes', config
['max_time_minutes']])
314 cmd
.extend(['--build_preference', 'ninja'])
316 if '--browser=cros' in config
['command']:
317 cmd
.extend(['--target_platform', 'cros'])
319 if os
.environ
[CROS_BOARD_ENV
] and os
.environ
[CROS_IP_ENV
]:
320 cmd
.extend(['--cros_board', os
.environ
[CROS_BOARD_ENV
]])
321 cmd
.extend(['--cros_remote_ip', os
.environ
[CROS_IP_ENV
]])
323 print 'Error: Cros build selected, but BISECT_CROS_IP or'\
324 'BISECT_CROS_BOARD undefined.'
328 if 'android' in config
['command']:
329 if 'android-chrome' in config
['command']:
330 cmd
.extend(['--target_platform', 'android-chrome'])
332 cmd
.extend(['--target_platform', 'android'])
335 cmd
.append('--use_goma')
337 if path_to_extra_src
:
338 cmd
.extend(['--extra_src', path_to_extra_src
])
341 cmd
.extend(['--debug_ignore_build', '--debug_ignore_sync',
342 '--debug_ignore_perf_test'])
343 cmd
= [str(c
) for c
in cmd
]
345 with
Goma(path_to_goma
) as goma
:
346 return_code
= subprocess
.call(cmd
)
349 print 'Error: bisect-perf-regression.py returned with error %d' %\
358 usage
= ('%prog [options] [-- chromium-options]\n'
359 'Used by a trybot to run the bisection script using the parameters'
360 ' provided in the run-bisect-perf-regression.cfg file.')
362 parser
= optparse
.OptionParser(usage
=usage
)
363 parser
.add_option('-w', '--working_directory',
365 help='A working directory to supply to the bisection '
366 'script, which will use it as the location to checkout '
367 'a copy of the chromium depot.')
368 parser
.add_option('-p', '--path_to_goma',
370 help='Path to goma directory. If this is supplied, goma '
371 'builds will be enabled.')
372 parser
.add_option('--extra_src',
374 help='Path to extra source file. If this is supplied, '
375 'bisect script will use this to override default behavior.')
376 parser
.add_option('--dry_run',
378 help='The script will perform the full bisect, but '
379 'without syncing, building, or running the performance '
381 (opts
, args
) = parser
.parse_args()
383 path_to_current_directory
= os
.path
.abspath(os
.path
.dirname(sys
.argv
[0]))
384 path_to_bisect_cfg
= os
.path
.join(path_to_current_directory
,
385 'run-bisect-perf-regression.cfg')
387 config
= _LoadConfigFile(path_to_bisect_cfg
)
389 # Check if the config is empty
390 config_has_values
= [v
for v
in config
.values() if v
]
392 if config
and config_has_values
:
393 if not opts
.working_directory
:
394 print 'Error: missing required parameter: --working_directory'
399 return _RunBisectionScript(config
, opts
.working_directory
,
400 path_to_current_directory
, opts
.path_to_goma
, opts
.extra_src
,
403 perf_cfg_files
= ['run-perf-test.cfg', os
.path
.join('..', 'third_party',
404 'WebKit', 'Tools', 'run-perf-test.cfg')]
406 for current_perf_cfg_file
in perf_cfg_files
:
407 path_to_perf_cfg
= os
.path
.join(
408 os
.path
.abspath(os
.path
.dirname(sys
.argv
[0])), current_perf_cfg_file
)
410 config
= _LoadConfigFile(path_to_perf_cfg
)
411 config_has_values
= [v
for v
in config
.values() if v
]
413 if config
and config_has_values
:
414 return _SetupAndRunPerformanceTest(config
, path_to_current_directory
,
417 print 'Error: Could not load config file. Double check your changes to '\
418 'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
423 if __name__
== '__main__':