From 19cbf13dc344795b0cd81169a26a5fcb3f885b04 Mon Sep 17 00:00:00 2001 From: "rmcilroy@chromium.org" Date: Wed, 10 Jul 2013 13:10:23 +0000 Subject: [PATCH] Add a simple tool which enables automated bisecting of codebase with manual test Creates a tool which enables bisection the codebase, with the user prompted to manually test a given revision and enter whether it is good or bad. Useful in tracking down the CL which caused a bug which can be triggered manually. BUG= Review URL: https://chromiumcodereview.appspot.com/18316008 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@210824 0039d316-1c4b-4281-b951-d872f2087c98 --- tools/bisect-manual-test.py | 52 ++++++++++++++++++ tools/bisect-perf-regression.py | 7 +-- tools/run-bisect-manual-test.py | 113 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 3 deletions(-) create mode 100755 tools/bisect-manual-test.py create mode 100755 tools/run-bisect-manual-test.py diff --git a/tools/bisect-manual-test.py b/tools/bisect-manual-test.py new file mode 100755 index 000000000000..62aa072c0352 --- /dev/null +++ b/tools/bisect-manual-test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Simple script which asks user to manually check result of bisection. + +Typically used as by the run-bisect-manual-test.py script. +""" + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry')) +from telemetry.core import browser_finder +from telemetry.core import browser_options + + +def _StartManualTest(options): + """Start browser then ask the user whether build is good or bad.""" + browser_to_create = browser_finder.FindBrowser(options) + print 'Starting browser: %s.' % options.browser_type + with browser_to_create.Create() as browser: + + # Loop until we get a response that we can parse. + while True: + sys.stderr.write('Revision is [(g)ood/(b)ad]: ') + response = raw_input() + if response and response in ('g', 'b'): + if response in ('g'): + print "RESULT manual_test: manual_test= 1" + else: + print "RESULT manual_test: manual_test= 0" + break + + browser.Close() + + +def main(): + usage = ('%prog [options]\n' + 'Starts browser with an optional url and asks user whether ' + 'revision is good or bad.\n') + + options = browser_options.BrowserOptions() + parser = options.CreateParser(usage) + options, args = parser.parse_args() + + _StartManualTest(options) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/bisect-perf-regression.py b/tools/bisect-perf-regression.py index 35ed89c89970..37eef69cb578 100755 --- a/tools/bisect-perf-regression.py +++ b/tools/bisect-perf-regression.py @@ -166,8 +166,10 @@ def CalculateTruncatedMean(data_set, truncate_percent): def CalculateStandardDeviation(v): - mean = CalculateTruncatedMean(v, 0.0) + if len(v) == 1: + return 0.0 + mean = CalculateTruncatedMean(v, 0.0) variances = [float(x) - mean for x in v] variances = [x * x for x in variances] variance = reduce(lambda x, y: float(x) + float(y), variances) / (len(v) - 1) @@ -253,8 +255,7 @@ def RunProcessAndRetrieveOutput(command): shell = IsWindows() proc = subprocess.Popen(command, shell=shell, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stdout=subprocess.PIPE) (output, _) = proc.communicate() diff --git a/tools/run-bisect-manual-test.py b/tools/run-bisect-manual-test.py new file mode 100755 index 000000000000..7b038db2bb29 --- /dev/null +++ b/tools/run-bisect-manual-test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Run Manual Test Bisect Tool""" + +import os +import subprocess +import sys + +CROS_BOARD_ENV = 'BISECT_CROS_BOARD' +CROS_IP_ENV = 'BISECT_CROS_IP' +_DIR_TOOLS_ROOT = os.path.abspath(os.path.dirname(__file__)) + +sys.path.append(os.path.join(_DIR_TOOLS_ROOT, 'telemetry')) +from telemetry.core import browser_options + + +def _RunBisectionScript(options): + """Attempts to execute src/tools/bisect-perf-regression.py with the parameters + passed in. + + Args: + options: The configuration options to pass to the bisect script. + + Returns: + 0 on success, otherwise 1. + """ + test_command = 'python %s --browser=%s' %\ + (os.path.join(_DIR_TOOLS_ROOT, 'bisect-manual-test.py'), + options.browser_type) + + cmd = ['python', os.path.join(_DIR_TOOLS_ROOT, 'bisect-perf-regression.py'), + '-c', test_command, + '-g', options.good_revision, + '-b', options.bad_revision, + '-m', 'manual_test/manual_test', + '-r', '1', + '--working_directory', options.working_directory, + '--build_preference', 'ninja', + '--use_goma'] + + if 'cros' in options.browser_type: + cmd.extend(['--target_platform', 'cros']) + + if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: + cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]]) + cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]]) + else: + print 'Error: Cros build selected, but BISECT_CROS_IP or'\ + 'BISECT_CROS_BOARD undefined.' + print + return 1 + elif 'android' in options.browser_type: + cmd.extend(['--target_platform', 'android']) + + cmd = [str(c) for c in cmd] + + return_code = subprocess.call(cmd) + + if return_code: + print 'Error: bisect-perf-regression.py returned with error %d' %\ + return_code + print + + return return_code + + +def main(): + usage = ('%prog [options]\n' + 'Used to run the bisection script with a manual test.') + + options = browser_options.BrowserOptions() + parser = options.CreateParser(usage) + + parser.add_option('-b', '--bad_revision', + type='str', + help='A bad revision to start bisection. ' + + 'Must be later than good revision. May be either a git' + + ' or svn revision.') + parser.add_option('-g', '--good_revision', + type='str', + help='A revision to start bisection where performance' + + ' test is known to pass. Must be earlier than the ' + + 'bad revision. May be either a git or svn revision.') + parser.add_option('-w', '--working_directory', + type='str', + help='A working directory to supply to the bisection ' + 'script, which will use it as the location to checkout ' + 'a copy of the chromium depot.') + options, args = parser.parse_args() + + error_msg = '' + if not options.browser_type: + error_msg += 'Error: missing required parameter: --browser\n' + if not options.working_directory: + error_msg += 'Error: missing required parameter: --working_directory\n' + if not options.good_revision: + error_msg += 'Error: missing required parameter: --good_revision\n' + if not options.bad_revision: + error_msg += 'Error: missing required parameter: --bad_revision\n' + + if error_msg: + print error_msg + parser.print_help() + return 1 + + return _RunBisectionScript(options) + + +if __name__ == '__main__': + sys.exit(main()) -- 2.11.4.GIT