tools/dromaeo_benchmark_runner/dromaeo_benchmark_runner.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2009 Google Inc. All Rights Reserved.
   4
   5 """Dromaeo benchmark automation script.
   6
   7 Script runs dromaeo tests in browsers specified by --browser switch and saves
   8 results to a spreadsheet on docs.google.com.
   9
  10 Prerequisites:
  11 1. Install Google Data APIs Python Client Library from
  12    http://code.google.com/p/gdata-python-client.
  13 2. Checkout Dromaeo benchmark from
  14    http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide
  15    local path to it in --dromaeo_home switch.
  16 3. Create a spreadsheet at http://docs.google.com and specify it's name in
  17    --spreadsheet switch
  18
  19 Benchmark results are presented in the following format:
  20 browser | date time
  21 test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error
  22 test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error
  23 ...
  24
  25 Here mij is mean run/s in individual dromaeo test i during benchmark run j,
  26 eij is error in individual dromaeo test i during benchmark run j.
  27
  28 Example usage:
  29 dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe"
  30     -b "C:\Program Files (x86)\Safari\safari.exe"
  31     -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1
  32     -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com
  33
  34 """
  35
  36 import getpass
  37 import json
  38 import os
  39 import re
  40 import subprocess
  41 import time
  42 import urlparse
  43 from optparse import OptionParser
  44 from BaseHTTPServer import HTTPServer
  45 import SimpleHTTPServer
  46 import gdata.spreadsheet.service
  47
  48 max_spreadsheet_columns = 20
  49 test_props = ['mean', 'error']
  50
  51 def ParseArguments():
  52   parser = OptionParser()
  53   parser.add_option("-b", "--browser",
  54                     action="append", dest="browsers",
  55                     help="list of browsers to test")
  56   parser.add_option("-n", "--run_count", dest="run_count", type="int",
  57                     default=5, help="number of runs")
  58   parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home",
  59                     help="directory with your dromaeo files")
  60   parser.add_option("-p", "--port", dest="port", type="int",
  61                     default=8080, help="http server port")
  62   parser.add_option("-f", "--filter", dest="filter",
  63                     default="dom", help="dromaeo suite filter")
  64   parser.add_option("-e", "--email", dest="email",
  65                     help="your google docs account")
  66   parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title",
  67                     default="dromaeo",
  68                     help="your google docs spreadsheet name")
  69
  70   options = parser.parse_args()[0]
  71
  72   if not options.dromaeo_home:
  73     raise Exception('please specify dromaeo_home')
  74
  75   return options
  76
  77
  78 def KillProcessByName(process_name):
  79   process = subprocess.Popen('wmic process get processid, executablepath',
  80                              stdout=subprocess.PIPE)
  81   stdout = str(process.communicate()[0])
  82   match = re.search(re.escape(process_name) + '\s+(\d+)', stdout)
  83   if match:
  84     pid = match.group(1)
  85     subprocess.call('taskkill /pid %s' % pid)
  86
  87
  88 class SpreadsheetWriter(object):
  89   "Utility class for storing benchmarking results in Google spreadsheets."
  90
  91   def __init__(self, email, spreadsheet_title):
  92     '''Login to google docs and search for spreadsheet'''
  93
  94     self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token")
  95     self.gd_client = gdata.spreadsheet.service.SpreadsheetsService()
  96
  97     authenticated = False
  98     if os.path.exists(self.token_file):
  99       token = ''
 100       try:
 101         file = open(self.token_file, 'r')
 102         token = file.read()
 103         file.close()
 104         self.gd_client.SetClientLoginToken(token)
 105         self.gd_client.GetSpreadsheetsFeed()
 106         authenticated = True
 107       except (IOError, gdata.service.RequestError):
 108         pass
 109     if not authenticated:
 110       self.gd_client.email = email
 111       self.gd_client.password = getpass.getpass('Password for %s: ' % email)
 112       self.gd_client.source = 'python robot for dromaeo'
 113       self.gd_client.ProgrammaticLogin()
 114       token = self.gd_client.GetClientLoginToken()
 115       try:
 116         file = open(self.token_file, 'w')
 117         file.write(token)
 118         file.close()
 119       except (IOError):
 120         pass
 121       os.chmod(self.token_file, 0600)
 122
 123     # Search for the spreadsheet with title = spreadsheet_title.
 124     spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed()
 125     for spreadsheet in spreadsheet_feed.entry:
 126       if spreadsheet.title.text == spreadsheet_title:
 127         self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1]
 128     if not self.spreadsheet_key:
 129       raise Exception('Spreadsheet %s not found' % spreadsheet_title)
 130
 131     # Get the key of the first worksheet in spreadsheet.
 132     worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key)
 133     self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1]
 134
 135   def _InsertRow(self, row):
 136     row = dict([('c' + str(i), row[i]) for i in xrange(len(row))])
 137     self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key)
 138
 139   def _InsertBlankRow(self):
 140     self._InsertRow('-' * self.columns_count)
 141
 142   def PrepareSpreadsheet(self, run_count):
 143     """Update cells in worksheet topmost row with service information.
 144
 145     Calculate column count corresponding to run_count and create worksheet
 146     column titles [c0, c1, ...] in the topmost row to speed up spreadsheet
 147     updates (it allows to insert a whole row with a single request)
 148     """
 149
 150     # Calculate the number of columns we need to present all test results.
 151     self.columns_count = (run_count + 2) * len(test_props)
 152     if self.columns_count > max_spreadsheet_columns:
 153       # Google spreadsheet has just max_spreadsheet_columns columns.
 154       max_run_count = max_spreadsheet_columns / len(test_props) - 2
 155       raise Exception('maximum run count is %i' % max_run_count)
 156     # Create worksheet column titles [c0, c1, ..., cn].
 157     for i in xrange(self.columns_count):
 158       self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key,
 159                                 self.worksheet_key)
 160
 161   def WriteColumnTitles(self, run_count):
 162     "Create titles for test results (mean 1, mean 2, ..., average mean, ...)"
 163     row = []
 164     for prop in test_props:
 165       row.append('')
 166       for i in xrange(run_count):
 167         row.append('%s %i' % (prop, i + 1))
 168       row.append('average ' + prop)
 169     self._InsertRow(row)
 170
 171   def WriteBrowserBenchmarkTitle(self, browser_name):
 172     "Create browser benchmark title (browser name, date time)"
 173     self._InsertBlankRow()
 174     self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')])
 175
 176   def WriteBrowserBenchmarkResults(self, test_name, test_data):
 177     "Insert a row with single test results"
 178     row = []
 179     for prop in test_props:
 180       if not row:
 181         row.append(test_name)
 182       else:
 183         row.append('')
 184       row.extend([str(x) for x in test_data[prop]])
 185       row.append(str(sum(test_data[prop]) / len(test_data[prop])))
 186     self._InsertRow(row)
 187
 188
 189 class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 190
 191   def do_POST(self):
 192     self.send_response(200)
 193     self.end_headers()
 194     self.wfile.write("<HTML>POST OK.<BR><BR>");
 195     length = int(self.headers.getheader('content-length'))
 196     parameters = urlparse.parse_qs(self.rfile.read(length))
 197     self.server.got_post = True
 198     self.server.post_data = parameters['data']
 199
 200
 201 class BenchmarkResults(object):
 202   "Storage class for dromaeo benchmark results"
 203
 204   def __init__(self):
 205     self.data = {}
 206
 207   def ProcessBrowserPostData(self, data):
 208     "Convert dromaeo test results in internal format"
 209     tests = json.loads(data[0])
 210     for test in tests:
 211       test_name = test['name']
 212       if test_name not in self.data:
 213         # Test is encountered for the first time.
 214         self.data[test_name] = dict([(prop, []) for prop in test_props])
 215       # Append current run results.
 216       for prop in test_props:
 217         value = -1
 218         if prop in test: value = test[prop] # workaround for Opera 10.5
 219         self.data[test_name][prop].append(value)
 220
 221
 222 def main():
 223   options = ParseArguments()
 224
 225   # Start sever with dromaeo.
 226   os.chdir(options.dromaeo_home)
 227   server = HTTPServer(('', options.port), DromaeoHandler)
 228
 229   # Open and prepare spreadsheet on google docs.
 230   spreadsheet_writer = SpreadsheetWriter(options.email,
 231                                          options.spreadsheet_title)
 232   spreadsheet_writer.PrepareSpreadsheet(options.run_count)
 233   spreadsheet_writer.WriteColumnTitles(options.run_count)
 234
 235   for browser in options.browsers:
 236     browser_name = os.path.splitext(os.path.basename(browser))[0]
 237     spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name)
 238     benchmark_results = BenchmarkResults()
 239     for run_number in xrange(options.run_count):
 240       print '%s run %i' % (browser_name, run_number + 1)
 241       # Run browser.
 242       test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % (
 243         options.port, options.filter)
 244       browser_process = subprocess.Popen('%s "%s"' % (browser, test_page))
 245       server.got_post = False
 246       server.post_data = None
 247       # Wait until POST request from browser.
 248       while not server.got_post:
 249         server.handle_request()
 250       benchmark_results.ProcessBrowserPostData(server.post_data)
 251       # Kill browser.
 252       KillProcessByName(browser)
 253       browser_process.wait()
 254
 255     # Insert test results into spreadsheet.
 256     for (test_name, test_data) in benchmark_results.data.iteritems():
 257       spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data)
 258
 259   server.socket.close()
 260
 261 if __name__ == '__main__':
 262   main()
 263