tools/dromaeo_benchmark_runner/dromaeo_benchmark_runner.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Dromaeo benchmark automation script.
   7
   8 Script runs dromaeo tests in browsers specified by --browser switch and saves
   9 results to a spreadsheet on docs.google.com.
  10
  11 Prerequisites:
  12 1. Install Google Data APIs Python Client Library from
  13    http://code.google.com/p/gdata-python-client.
  14 2. Checkout Dromaeo benchmark from
  15    http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide
  16    local path to it in --dromaeo_home switch.
  17 3. Create a spreadsheet at http://docs.google.com and specify its name in
  18    --spreadsheet switch
  19
  20 Benchmark results are presented in the following format:
  21 browser | date time
  22 test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error
  23 test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error
  24 ...
  25
  26 Here mij is mean run/s in individual dromaeo test i during benchmark run j,
  27 eij is error in individual dromaeo test i during benchmark run j.
  28
  29 Example usage:
  30 dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe"
  31     -b "C:\Program Files (x86)\Safari\safari.exe"
  32     -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1
  33     -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com
  34
  35 """
  36
  37 import getpass
  38 import json
  39 import os
  40 import re
  41 import subprocess
  42 import time
  43 import urlparse
  44 from optparse import OptionParser
  45 from BaseHTTPServer import HTTPServer
  46 import SimpleHTTPServer
  47 import gdata.spreadsheet.service
  48
  49 max_spreadsheet_columns = 20
  50 test_props = ['mean', 'error']
  51
  52
  53 def ParseArguments():
  54   parser = OptionParser()
  55   parser.add_option("-b", "--browser",
  56                     action="append", dest="browsers",
  57                     help="list of browsers to test")
  58   parser.add_option("-n", "--run_count", dest="run_count", type="int",
  59                     default=5, help="number of runs")
  60   parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home",
  61                     help="directory with your dromaeo files")
  62   parser.add_option("-p", "--port", dest="port", type="int",
  63                     default=8080, help="http server port")
  64   parser.add_option("-f", "--filter", dest="filter",
  65                     default="dom", help="dromaeo suite filter")
  66   parser.add_option("-e", "--email", dest="email",
  67                     help="your google docs account")
  68   parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title",
  69                     default="dromaeo",
  70                     help="your google docs spreadsheet name")
  71
  72   options = parser.parse_args()[0]
  73
  74   if not options.dromaeo_home:
  75     raise Exception('please specify dromaeo_home')
  76
  77   return options
  78
  79
  80 def KillProcessByName(process_name):
  81   process = subprocess.Popen('wmic process get processid, executablepath',
  82                              stdout=subprocess.PIPE)
  83   stdout = str(process.communicate()[0])
  84   match = re.search(re.escape(process_name) + '\s+(\d+)', stdout)
  85   if match:
  86     pid = match.group(1)
  87     subprocess.call('taskkill /pid %s' % pid)
  88
  89
  90 class SpreadsheetWriter(object):
  91   "Utility class for storing benchmarking results in Google spreadsheets."
  92
  93   def __init__(self, email, spreadsheet_title):
  94     '''Login to google docs and search for spreadsheet'''
  95
  96     self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token")
  97     self.gd_client = gdata.spreadsheet.service.SpreadsheetsService()
  98
  99     authenticated = False
 100     if os.path.exists(self.token_file):
 101       token = ''
 102       try:
 103         file = open(self.token_file, 'r')
 104         token = file.read()
 105         file.close()
 106         self.gd_client.SetClientLoginToken(token)
 107         self.gd_client.GetSpreadsheetsFeed()
 108         authenticated = True
 109       except (IOError, gdata.service.RequestError):
 110         pass
 111     if not authenticated:
 112       self.gd_client.email = email
 113       self.gd_client.password = getpass.getpass('Password for %s: ' % email)
 114       self.gd_client.source = 'python robot for dromaeo'
 115       self.gd_client.ProgrammaticLogin()
 116       token = self.gd_client.GetClientLoginToken()
 117       try:
 118         file = open(self.token_file, 'w')
 119         file.write(token)
 120         file.close()
 121       except (IOError):
 122         pass
 123       os.chmod(self.token_file, 0600)
 124
 125     # Search for the spreadsheet with title = spreadsheet_title.
 126     spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed()
 127     for spreadsheet in spreadsheet_feed.entry:
 128       if spreadsheet.title.text == spreadsheet_title:
 129         self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1]
 130     if not self.spreadsheet_key:
 131       raise Exception('Spreadsheet %s not found' % spreadsheet_title)
 132
 133     # Get the key of the first worksheet in spreadsheet.
 134     worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key)
 135     self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1]
 136
 137   def _InsertRow(self, row):
 138     row = dict([('c' + str(i), row[i]) for i in xrange(len(row))])
 139     self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key)
 140
 141   def _InsertBlankRow(self):
 142     self._InsertRow('-' * self.columns_count)
 143
 144   def PrepareSpreadsheet(self, run_count):
 145     """Update cells in worksheet topmost row with service information.
 146
 147     Calculate column count corresponding to run_count and create worksheet
 148     column titles [c0, c1, ...] in the topmost row to speed up spreadsheet
 149     updates (it allows to insert a whole row with a single request)
 150     """
 151
 152     # Calculate the number of columns we need to present all test results.
 153     self.columns_count = (run_count + 2) * len(test_props)
 154     if self.columns_count > max_spreadsheet_columns:
 155       # Google spreadsheet has just max_spreadsheet_columns columns.
 156       max_run_count = max_spreadsheet_columns / len(test_props) - 2
 157       raise Exception('maximum run count is %i' % max_run_count)
 158     # Create worksheet column titles [c0, c1, ..., cn].
 159     for i in xrange(self.columns_count):
 160       self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key,
 161                                 self.worksheet_key)
 162
 163   def WriteColumnTitles(self, run_count):
 164     "Create titles for test results (mean 1, mean 2, ..., average mean, ...)"
 165     row = []
 166     for prop in test_props:
 167       row.append('')
 168       for i in xrange(run_count):
 169         row.append('%s %i' % (prop, i + 1))
 170       row.append('average ' + prop)
 171     self._InsertRow(row)
 172
 173   def WriteBrowserBenchmarkTitle(self, browser_name):
 174     "Create browser benchmark title (browser name, date time)"
 175     self._InsertBlankRow()
 176     self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')])
 177
 178   def WriteBrowserBenchmarkResults(self, test_name, test_data):
 179     "Insert a row with single test results"
 180     row = []
 181     for prop in test_props:
 182       if not row:
 183         row.append(test_name)
 184       else:
 185         row.append('')
 186       row.extend([str(x) for x in test_data[prop]])
 187       row.append(str(sum(test_data[prop]) / len(test_data[prop])))
 188     self._InsertRow(row)
 189
 190
 191 class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 192
 193   def do_POST(self):
 194     self.send_response(200)
 195     self.end_headers()
 196     self.wfile.write("<HTML>POST OK.<BR><BR>");
 197     length = int(self.headers.getheader('content-length'))
 198     parameters = urlparse.parse_qs(self.rfile.read(length))
 199     self.server.got_post = True
 200     self.server.post_data = parameters['data']
 201
 202
 203 class BenchmarkResults(object):
 204   "Storage class for dromaeo benchmark results"
 205
 206   def __init__(self):
 207     self.data = {}
 208
 209   def ProcessBrowserPostData(self, data):
 210     "Convert dromaeo test results in internal format"
 211     tests = json.loads(data[0])
 212     for test in tests:
 213       test_name = test['name']
 214       if test_name not in self.data:
 215         # Test is encountered for the first time.
 216         self.data[test_name] = dict([(prop, []) for prop in test_props])
 217       # Append current run results.
 218       for prop in test_props:
 219         value = -1
 220         if prop in test: value = test[prop] # workaround for Opera 10.5
 221         self.data[test_name][prop].append(value)
 222
 223
 224 def main():
 225   options = ParseArguments()
 226
 227   # Start sever with dromaeo.
 228   os.chdir(options.dromaeo_home)
 229   server = HTTPServer(('', options.port), DromaeoHandler)
 230
 231   # Open and prepare spreadsheet on google docs.
 232   spreadsheet_writer = SpreadsheetWriter(options.email,
 233                                          options.spreadsheet_title)
 234   spreadsheet_writer.PrepareSpreadsheet(options.run_count)
 235   spreadsheet_writer.WriteColumnTitles(options.run_count)
 236
 237   for browser in options.browsers:
 238     browser_name = os.path.splitext(os.path.basename(browser))[0]
 239     spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name)
 240     benchmark_results = BenchmarkResults()
 241     for run_number in xrange(options.run_count):
 242       print '%s run %i' % (browser_name, run_number + 1)
 243       # Run browser.
 244       test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % (
 245         options.port, options.filter)
 246       browser_process = subprocess.Popen('%s "%s"' % (browser, test_page))
 247       server.got_post = False
 248       server.post_data = None
 249       # Wait until POST request from browser.
 250       while not server.got_post:
 251         server.handle_request()
 252       benchmark_results.ProcessBrowserPostData(server.post_data)
 253       # Kill browser.
 254       KillProcessByName(browser)
 255       browser_process.wait()
 256
 257     # Insert test results into spreadsheet.
 258     for (test_name, test_data) in benchmark_results.data.iteritems():
 259       spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data)
 260
 261   server.socket.close()
 262   return 0
 263
 264
 265 if __name__ == '__main__':
 266   sys.exit(main())