ppapi/native_client/tools/browser_tester/browser_tester.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import glob
   7 import optparse
   8 import os.path
   9 import socket
  10 import sys
  11 import thread
  12 import time
  13 import urllib
  14
  15 # Allow the import of third party modules
  16 script_dir = os.path.dirname(os.path.abspath(__file__))
  17 sys.path.insert(0, os.path.join(script_dir, '../../../../third_party/'))
  18 sys.path.insert(0, os.path.join(script_dir, '../../../../tools/valgrind/'))
  19 sys.path.insert(0, os.path.join(script_dir, '../../../../testing/'))
  20
  21 import browsertester.browserlauncher
  22 import browsertester.rpclistener
  23 import browsertester.server
  24
  25 import memcheck_analyze
  26 import tsan_analyze
  27
  28 import test_env
  29
  30 def BuildArgParser():
  31   usage = 'usage: %prog [options]'
  32   parser = optparse.OptionParser(usage)
  33
  34   parser.add_option('-p', '--port', dest='port', action='store', type='int',
  35                     default='0', help='The TCP port the server will bind to. '
  36                     'The default is to pick an unused port number.')
  37   parser.add_option('--browser_path', dest='browser_path', action='store',
  38                     type='string', default=None,
  39                     help='Use the browser located here.')
  40   parser.add_option('--map_file', dest='map_files', action='append',
  41                     type='string', nargs=2, default=[],
  42                     metavar='DEST SRC',
  43                     help='Add file SRC to be served from the HTTP server, '
  44                     'to be made visible under the path DEST.')
  45   parser.add_option('--serving_dir', dest='serving_dirs', action='append',
  46                     type='string', default=[],
  47                     metavar='DIRNAME',
  48                     help='Add directory DIRNAME to be served from the HTTP '
  49                     'server to be made visible under the root.')
  50   parser.add_option('--output_dir', dest='output_dir', action='store',
  51                     type='string', default=None,
  52                     metavar='DIRNAME',
  53                     help='Set directory DIRNAME to be the output directory '
  54                     'when POSTing data to the server. NOTE: if this flag is '
  55                     'not set, POSTs will fail.')
  56   parser.add_option('--test_arg', dest='test_args', action='append',
  57                     type='string', nargs=2, default=[],
  58                     metavar='KEY VALUE',
  59                     help='Parameterize the test with a key/value pair.')
  60   parser.add_option('--redirect_url', dest='map_redirects', action='append',
  61                     type='string', nargs=2, default=[],
  62                     metavar='DEST SRC',
  63                     help='Add a redirect to the HTTP server, '
  64                     'requests for SRC will result in a redirect (302) to DEST.')
  65   parser.add_option('-f', '--file', dest='files', action='append',
  66                     type='string', default=[],
  67                     metavar='FILENAME',
  68                     help='Add a file to serve from the HTTP server, to be '
  69                     'made visible in the root directory.  '
  70                     '"--file path/to/foo.html" is equivalent to '
  71                     '"--map_file foo.html path/to/foo.html"')
  72   parser.add_option('--mime_type', dest='mime_types', action='append',
  73                     type='string', nargs=2, default=[], metavar='DEST SRC',
  74                     help='Map file extension SRC to MIME type DEST when '
  75                     'serving it from the HTTP server.')
  76   parser.add_option('-u', '--url', dest='url', action='store',
  77                     type='string', default=None,
  78                     help='The webpage to load.')
  79   parser.add_option('--ppapi_plugin', dest='ppapi_plugin', action='store',
  80                     type='string', default=None,
  81                     help='Use the browser plugin located here.')
  82   parser.add_option('--ppapi_plugin_mimetype', dest='ppapi_plugin_mimetype',
  83                     action='store', type='string', default='application/x-nacl',
  84                     help='Associate this mimetype with the browser plugin. '
  85                     'Unused if --ppapi_plugin is not specified.')
  86   parser.add_option('--sel_ldr', dest='sel_ldr', action='store',
  87                     type='string', default=None,
  88                     help='Use the sel_ldr located here.')
  89   parser.add_option('--sel_ldr_bootstrap', dest='sel_ldr_bootstrap',
  90                     action='store', type='string', default=None,
  91                     help='Use the bootstrap loader located here.')
  92   parser.add_option('--irt_library', dest='irt_library', action='store',
  93                     type='string', default=None,
  94                     help='Use the integrated runtime (IRT) library '
  95                     'located here.')
  96   parser.add_option('--interactive', dest='interactive', action='store_true',
  97                     default=False, help='Do not quit after testing is done. '
  98                     'Handy for iterative development.  Disables timeout.')
  99   parser.add_option('--debug', dest='debug', action='store_true', default=False,
 100                     help='Request debugging output from browser.')
 101   parser.add_option('--timeout', dest='timeout', action='store', type='float',
 102                     default=5.0,
 103                     help='The maximum amount of time to wait, in seconds, for '
 104                     'the browser to make a request. The timer resets with each '
 105                     'request.')
 106   parser.add_option('--hard_timeout', dest='hard_timeout', action='store',
 107                     type='float', default=None,
 108                     help='The maximum amount of time to wait, in seconds, for '
 109                     'the entire test.  This will kill runaway tests. ')
 110   parser.add_option('--allow_404', dest='allow_404', action='store_true',
 111                     default=False,
 112                     help='Allow 404s to occur without failing the test.')
 113   parser.add_option('-b', '--bandwidth', dest='bandwidth', action='store',
 114                     type='float', default='0.0',
 115                     help='The amount of bandwidth (megabits / second) to '
 116                     'simulate between the client and the server. This used for '
 117                     'replies with file payloads. All other responses are '
 118                     'assumed to be short. Bandwidth values <= 0.0 are assumed '
 119                     'to mean infinite bandwidth.')
 120   parser.add_option('--extension', dest='browser_extensions', action='append',
 121                     type='string', default=[],
 122                     help='Load the browser extensions located at the list of '
 123                     'paths. Note: this currently only works with the Chrome '
 124                     'browser.')
 125   parser.add_option('--tool', dest='tool', action='store',
 126                     type='string', default=None,
 127                     help='Run tests under a tool.')
 128   parser.add_option('--browser_flag', dest='browser_flags', action='append',
 129                     type='string', default=[],
 130                     help='Additional flags for the chrome command.')
 131   parser.add_option('--enable_ppapi_dev', dest='enable_ppapi_dev',
 132                     action='store', type='int', default=1,
 133                     help='Enable/disable PPAPI Dev interfaces while testing.')
 134   parser.add_option('--nacl_exe_stdin', dest='nacl_exe_stdin',
 135                     type='string', default=None,
 136                     help='Redirect standard input of NaCl executable.')
 137   parser.add_option('--nacl_exe_stdout', dest='nacl_exe_stdout',
 138                     type='string', default=None,
 139                     help='Redirect standard output of NaCl executable.')
 140   parser.add_option('--nacl_exe_stderr', dest='nacl_exe_stderr',
 141                     type='string', default=None,
 142                     help='Redirect standard error of NaCl executable.')
 143   parser.add_option('--expect_browser_process_crash',
 144                     dest='expect_browser_process_crash',
 145                     action='store_true',
 146                     help='Do not signal a failure if the browser process '
 147                     'crashes')
 148   parser.add_option('--enable_crash_reporter', dest='enable_crash_reporter',
 149                     action='store_true', default=False,
 150                     help='Force crash reporting on.')
 151   parser.add_option('--enable_sockets', dest='enable_sockets',
 152                     action='store_true', default=False,
 153                     help='Pass --allow-nacl-socket-api=<host> to Chrome, where '
 154                     '<host> is the name of the browser tester\'s web server.')
 155
 156   return parser
 157
 158
 159 def ProcessToolLogs(options, logs_dir):
 160   if options.tool == 'memcheck':
 161     analyzer = memcheck_analyze.MemcheckAnalyzer('', use_gdb=True)
 162     logs_wildcard = 'xml.*'
 163   elif options.tool == 'tsan':
 164     analyzer = tsan_analyze.TsanAnalyzer(use_gdb=True)
 165     logs_wildcard = 'log.*'
 166   files = glob.glob(os.path.join(logs_dir, logs_wildcard))
 167   retcode = analyzer.Report(files, options.url)
 168   return retcode
 169
 170
 171 # An exception that indicates possible flake.
 172 class RetryTest(Exception):
 173   pass
 174
 175
 176 def DumpNetLog(netlog):
 177   sys.stdout.write('\n')
 178   if not os.path.isfile(netlog):
 179     sys.stdout.write('Cannot find netlog, did Chrome actually launch?\n')
 180   else:
 181     sys.stdout.write('Netlog exists (%d bytes).\n' % os.path.getsize(netlog))
 182     sys.stdout.write('Dumping it to stdout.\n\n\n')
 183     sys.stdout.write(open(netlog).read())
 184     sys.stdout.write('\n\n\n')
 185
 186
 187 # Try to discover the real IP address of this machine.  If we can't figure it
 188 # out, fall back to localhost.
 189 # A windows bug makes using the loopback interface flaky in rare cases.
 190 # http://code.google.com/p/chromium/issues/detail?id=114369
 191 def GetHostName():
 192   host = 'localhost'
 193   try:
 194     host = socket.gethostbyname(socket.gethostname())
 195   except Exception:
 196     pass
 197   if host == '0.0.0.0':
 198     host = 'localhost'
 199   return host
 200
 201
 202 def RunTestsOnce(url, options):
 203   # Set the default here so we're assured hard_timeout will be defined.
 204   # Tests, such as run_inbrowser_trusted_crash_in_startup_test, may not use the
 205   # RunFromCommand line entry point - and otherwise get stuck in an infinite
 206   # loop when something goes wrong and the hard timeout is not set.
 207   # http://code.google.com/p/chromium/issues/detail?id=105406
 208   if options.hard_timeout is None:
 209     options.hard_timeout = options.timeout * 4
 210
 211   options.files.append(os.path.join(script_dir, 'browserdata', 'nacltest.js'))
 212
 213   # Setup the environment with the setuid sandbox path.
 214   env.update(test_env.get_sandbox_env(sys.argv, os.environ))
 215
 216   # Create server
 217   host = GetHostName()
 218   try:
 219     server = browsertester.server.Create(host, options.port)
 220   except Exception:
 221     sys.stdout.write('Could not bind %r, falling back to localhost.\n' % host)
 222     server = browsertester.server.Create('localhost', options.port)
 223
 224   # If port 0 has been requested, an arbitrary port will be bound so we need to
 225   # query it.  Older version of Python do not set server_address correctly when
 226   # The requested port is 0 so we need to break encapsulation and query the
 227   # socket directly.
 228   host, port = server.socket.getsockname()
 229
 230   file_mapping = dict(options.map_files)
 231   for filename in options.files:
 232     file_mapping[os.path.basename(filename)] = filename
 233   for server_path, real_path in file_mapping.iteritems():
 234     if not os.path.exists(real_path):
 235       raise AssertionError('\'%s\' does not exist.' % real_path)
 236   mime_types = {}
 237   for ext, mime_type in options.mime_types:
 238     mime_types['.' + ext] = mime_type
 239
 240   def ShutdownCallback():
 241     server.TestingEnded()
 242     close_browser = options.tool is not None and not options.interactive
 243     return close_browser
 244
 245   listener = browsertester.rpclistener.RPCListener(ShutdownCallback)
 246   server.Configure(file_mapping,
 247                    dict(options.map_redirects),
 248                    mime_types,
 249                    options.allow_404,
 250                    options.bandwidth,
 251                    listener,
 252                    options.serving_dirs,
 253                    options.output_dir)
 254
 255   browser = browsertester.browserlauncher.ChromeLauncher(options)
 256
 257   full_url = 'http://%s:%d/%s' % (host, port, url)
 258   if len(options.test_args) > 0:
 259     full_url += '?' + urllib.urlencode(options.test_args)
 260   browser.Run(full_url, host, port)
 261   server.TestingBegun(0.125)
 262
 263   # In Python 2.5, server.handle_request may block indefinitely.  Serving pages
 264   # is done in its own thread so the main thread can time out as needed.
 265   def Serve():
 266     while server.test_in_progress or options.interactive:
 267       server.handle_request()
 268   thread.start_new_thread(Serve, ())
 269
 270   tool_failed = False
 271   time_started = time.time()
 272
 273   def HardTimeout(total_time):
 274     return total_time >= 0.0 and time.time() - time_started >= total_time
 275
 276   try:
 277     while server.test_in_progress or options.interactive:
 278       if not browser.IsRunning():
 279         if options.expect_browser_process_crash:
 280           break
 281         listener.ServerError('Browser process ended during test '
 282                              '(return code %r)' % browser.GetReturnCode())
 283         # If Chrome exits prematurely without making a single request to the
 284         # web server, this is probally a Chrome crash-on-launch bug not related
 285         # to the test at hand.  Retry, unless we're in interactive mode.  In
 286         # interactive mode the user may manually close the browser, so don't
 287         # retry (it would just be annoying.)
 288         if not server.received_request and not options.interactive:
 289           raise RetryTest('Chrome failed to launch.')
 290         else:
 291           break
 292       elif not options.interactive and server.TimedOut(options.timeout):
 293         js_time = server.TimeSinceJSHeartbeat()
 294         err = 'Did not hear from the test for %.1f seconds.' % options.timeout
 295         err += '\nHeard from Javascript %.1f seconds ago.' % js_time
 296         if js_time > 2.0:
 297           err += '\nThe renderer probably hung or crashed.'
 298         else:
 299           err += '\nThe test probably did not get a callback that it expected.'
 300         listener.ServerError(err)
 301         if not server.received_request:
 302           raise RetryTest('Chrome hung before running the test.')
 303         break
 304       elif not options.interactive and HardTimeout(options.hard_timeout):
 305         listener.ServerError('The test took over %.1f seconds.  This is '
 306                              'probably a runaway test.' % options.hard_timeout)
 307         break
 308       else:
 309         # If Python 2.5 support is dropped, stick server.handle_request() here.
 310         time.sleep(0.125)
 311
 312     if options.tool:
 313       sys.stdout.write('##################### Waiting for the tool to exit\n')
 314       browser.WaitForProcessDeath()
 315       sys.stdout.write('##################### Processing tool logs\n')
 316       tool_failed = ProcessToolLogs(options, browser.tool_log_dir)
 317
 318   finally:
 319     try:
 320       if listener.ever_failed and not options.interactive:
 321         if not server.received_request:
 322           sys.stdout.write('\nNo URLs were served by the test runner. It is '
 323                            'unlikely this test failure has anything to do with '
 324                            'this particular test.\n')
 325           DumpNetLog(browser.NetLogName())
 326     except Exception:
 327       listener.ever_failed = 1
 328     # Try to let the browser clean itself up normally before killing it.
 329     sys.stdout.write('##################### Terminating the browser\n')
 330     browser.WaitForProcessDeath()
 331     if browser.IsRunning():
 332       sys.stdout.write('##################### TERM failed, KILLING\n')
 333     # Always call Cleanup; it kills the process, but also removes the
 334     # user-data-dir.
 335     browser.Cleanup()
 336     # We avoid calling server.server_close() here because it causes
 337     # the HTTP server thread to exit uncleanly with an EBADF error,
 338     # which adds noise to the logs (though it does not cause the test
 339     # to fail).  server_close() does not attempt to tell the server
 340     # loop to shut down before closing the socket FD it is
 341     # select()ing.  Since we are about to exit, we don't really need
 342     # to close the socket FD.
 343
 344   if tool_failed:
 345     return 2
 346   elif listener.ever_failed:
 347     return 1
 348   else:
 349     return 0
 350
 351
 352 # This is an entrypoint for tests that treat the browser tester as a Python
 353 # library rather than an opaque script.
 354 # (e.g. run_inbrowser_trusted_crash_in_startup_test)
 355 def Run(url, options):
 356   result = 1
 357   attempt = 1
 358   while True:
 359     try:
 360       result = RunTestsOnce(url, options)
 361       if result:
 362         # Currently (2013/11/15) nacl_integration is fairly flaky and there is
 363         # not enough time to look into it.  Retry if the test fails for any
 364         # reason.  Note that in general this test runner tries to only retry
 365         # when a known flake is encountered.  (See the other raise
 366         # RetryTest(..)s in this file.)  This blanket retry means that those
 367         # other cases could be removed without changing the behavior of the test
 368         # runner, but it is hoped that this blanket retry will eventually be
 369         # unnecessary and subsequently removed.  The more precise retries have
 370         # been left in place to preserve the knowledge.
 371         raise RetryTest('HACK retrying failed test.')
 372       break
 373     except RetryTest:
 374       # Only retry once.
 375       if attempt < 2:
 376         sys.stdout.write('\n@@@STEP_WARNINGS@@@\n')
 377         sys.stdout.write('WARNING: suspected flake, retrying test!\n\n')
 378         attempt += 1
 379         continue
 380       else:
 381         sys.stdout.write('\nWARNING: failed too many times, not retrying.\n\n')
 382         result = 1
 383         break
 384   return result
 385
 386
 387 def RunFromCommandLine():
 388   parser = BuildArgParser()
 389   options, args = parser.parse_args()
 390
 391   if len(args) != 0:
 392     print args
 393     parser.error('Invalid arguments')
 394
 395   # Validate the URL
 396   url = options.url
 397   if url is None:
 398     parser.error('Must specify a URL')
 399
 400   return Run(url, options)
 401
 402
 403 if __name__ == '__main__':
 404   sys.exit(RunFromCommandLine())