monitor/cherrypy/test/benchmark.py

   1 """CherryPy Benchmark Tool
   2
   3     Usage:
   4         benchmark.py --null --notests --help --cpmodpy --modpython --ab=path --apache=path
   5
   6     --null:        use a null Request object (to bench the HTTP server only)
   7     --notests:     start the server but do not run the tests; this allows
   8                    you to check the tested pages with a browser
   9     --help:        show this help message
  10     --cpmodpy:     run tests via apache on 8080 (with the builtin _cpmodpy)
  11     --modpython:   run tests via apache on 8080 (with modpython_gateway)
  12     --ab=path:     Use the ab script/executable at 'path' (see below)
  13     --apache=path: Use the apache script/exe at 'path' (see below)
  14
  15     To run the benchmarks, the Apache Benchmark tool "ab" must either be on
  16     your system path, or specified via the --ab=path option.
  17
  18     To run the modpython tests, the "apache" executable or script must be
  19     on your system path, or provided via the --apache=path option. On some
  20     platforms, "apache" may be called "apachectl" or "apache2ctl"--create
  21     a symlink to them if needed.
  22 """
  23
  24 import getopt
  25 import os
  26 curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
  27
  28 import re
  29 import sys
  30 import time
  31 import traceback
  32
  33 import cherrypy
  34 from cherrypy._cpcompat import ntob
  35 from cherrypy import _cperror, _cpmodpy
  36 from cherrypy.lib import httputil
  37
  38
  39 AB_PATH = ""
  40 APACHE_PATH = "apache"
  41 SCRIPT_NAME = "/cpbench/users/rdelon/apps/blog"
  42
  43 __all__ = ['ABSession', 'Root', 'print_report',
  44            'run_standard_benchmarks', 'safe_threads',
  45            'size_report', 'startup', 'thread_report',
  46            ]
  47
  48 size_cache = {}
  49
  50 class Root:
  51
  52     def index(self):
  53         return """<html>
  54 <head>
  55     <title>CherryPy Benchmark</title>
  56 </head>
  57 <body>
  58     <ul>
  59         <li><a href="hello">Hello, world! (14 byte dynamic)</a></li>
  60         <li><a href="static/index.html">Static file (14 bytes static)</a></li>
  61         <li><form action="sizer">Response of length:
  62             <input type='text' name='size' value='10' /></form>
  63         </li>
  64     </ul>
  65 </body>
  66 </html>"""
  67     index.exposed = True
  68
  69     def hello(self):
  70         return "Hello, world\r\n"
  71     hello.exposed = True
  72
  73     def sizer(self, size):
  74         resp = size_cache.get(size, None)
  75         if resp is None:
  76             size_cache[size] = resp = "X" * int(size)
  77         return resp
  78     sizer.exposed = True
  79
  80
  81 cherrypy.config.update({
  82     'log.error.file': '',
  83     'environment': 'production',
  84     'server.socket_host': '127.0.0.1',
  85     'server.socket_port': 8080,
  86     'server.max_request_header_size': 0,
  87     'server.max_request_body_size': 0,
  88     'engine.deadlock_poll_freq': 0,
  89     })
  90
  91 # Cheat mode on ;)
  92 del cherrypy.config['tools.log_tracebacks.on']
  93 del cherrypy.config['tools.log_headers.on']
  94 del cherrypy.config['tools.trailing_slash.on']
  95
  96 appconf = {
  97     '/static': {
  98         'tools.staticdir.on': True,
  99         'tools.staticdir.dir': 'static',
 100         'tools.staticdir.root': curdir,
 101         },
 102     }
 103 app = cherrypy.tree.mount(Root(), SCRIPT_NAME, appconf)
 104
 105
 106 class NullRequest:
 107     """A null HTTP request class, returning 200 and an empty body."""
 108
 109     def __init__(self, local, remote, scheme="http"):
 110         pass
 111
 112     def close(self):
 113         pass
 114
 115     def run(self, method, path, query_string, protocol, headers, rfile):
 116         cherrypy.response.status = "200 OK"
 117         cherrypy.response.header_list = [("Content-Type", 'text/html'),
 118                                          ("Server", "Null CherryPy"),
 119                                          ("Date", httputil.HTTPDate()),
 120                                          ("Content-Length", "0"),
 121                                          ]
 122         cherrypy.response.body = [""]
 123         return cherrypy.response
 124
 125
 126 class NullResponse:
 127     pass
 128
 129
 130 class ABSession:
 131     """A session of 'ab', the Apache HTTP server benchmarking tool.
 132
 133 Example output from ab:
 134
 135 This is ApacheBench, Version 2.0.40-dev <$Revision: 1.121.2.1 $> apache-2.0
 136 Copyright (c) 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
 137 Copyright (c) 1998-2002 The Apache Software Foundation, http://www.apache.org/
 138
 139 Benchmarking 127.0.0.1 (be patient)
 140 Completed 100 requests
 141 Completed 200 requests
 142 Completed 300 requests
 143 Completed 400 requests
 144 Completed 500 requests
 145 Completed 600 requests
 146 Completed 700 requests
 147 Completed 800 requests
 148 Completed 900 requests
 149
 150
 151 Server Software:        CherryPy/3.1beta
 152 Server Hostname:        127.0.0.1
 153 Server Port:            8080
 154
 155 Document Path:          /static/index.html
 156 Document Length:        14 bytes
 157
 158 Concurrency Level:      10
 159 Time taken for tests:   9.643867 seconds
 160 Complete requests:      1000
 161 Failed requests:        0
 162 Write errors:           0
 163 Total transferred:      189000 bytes
 164 HTML transferred:       14000 bytes
 165 Requests per second:    103.69 [#/sec] (mean)
 166 Time per request:       96.439 [ms] (mean)
 167 Time per request:       9.644 [ms] (mean, across all concurrent requests)
 168 Transfer rate:          19.08 [Kbytes/sec] received
 169
 170 Connection Times (ms)
 171               min  mean[+/-sd] median   max
 172 Connect:        0    0   2.9      0      10
 173 Processing:    20   94   7.3     90     130
 174 Waiting:        0   43  28.1     40     100
 175 Total:         20   95   7.3    100     130
 176
 177 Percentage of the requests served within a certain time (ms)
 178   50%    100
 179   66%    100
 180   75%    100
 181   80%    100
 182   90%    100
 183   95%    100
 184   98%    100
 185   99%    110
 186  100%    130 (longest request)
 187 Finished 1000 requests
 188 """
 189
 190     parse_patterns = [('complete_requests', 'Completed',
 191                        ntob(r'^Complete requests:\s*(\d+)')),
 192                       ('failed_requests', 'Failed',
 193                        ntob(r'^Failed requests:\s*(\d+)')),
 194                       ('requests_per_second', 'req/sec',
 195                        ntob(r'^Requests per second:\s*([0-9.]+)')),
 196                       ('time_per_request_concurrent', 'msec/req',
 197                        ntob(r'^Time per request:\s*([0-9.]+).*concurrent requests\)$')),
 198                       ('transfer_rate', 'KB/sec',
 199                        ntob(r'^Transfer rate:\s*([0-9.]+)')),
 200                       ]
 201
 202     def __init__(self, path=SCRIPT_NAME + "/hello", requests=1000, concurrency=10):
 203         self.path = path
 204         self.requests = requests
 205         self.concurrency = concurrency
 206
 207     def args(self):
 208         port = cherrypy.server.socket_port
 209         assert self.concurrency > 0
 210         assert self.requests > 0
 211         # Don't use "localhost".
 212         # Cf http://mail.python.org/pipermail/python-win32/2008-March/007050.html
 213         return ("-k -n %s -c %s http://127.0.0.1:%s%s" %
 214                 (self.requests, self.concurrency, port, self.path))
 215
 216     def run(self):
 217         # Parse output of ab, setting attributes on self
 218         try:
 219             self.output = _cpmodpy.read_process(AB_PATH or "ab", self.args())
 220         except:
 221             print(_cperror.format_exc())
 222             raise
 223
 224         for attr, name, pattern in self.parse_patterns:
 225             val = re.search(pattern, self.output, re.MULTILINE)
 226             if val:
 227                 val = val.group(1)
 228                 setattr(self, attr, val)
 229             else:
 230                 setattr(self, attr, None)
 231
 232
 233 safe_threads = (25, 50, 100, 200, 400)
 234 if sys.platform in ("win32",):
 235     # For some reason, ab crashes with > 50 threads on my Win2k laptop.
 236     safe_threads = (10, 20, 30, 40, 50)
 237
 238
 239 def thread_report(path=SCRIPT_NAME + "/hello", concurrency=safe_threads):
 240     sess = ABSession(path)
 241     attrs, names, patterns = list(zip(*sess.parse_patterns))
 242     avg = dict.fromkeys(attrs, 0.0)
 243
 244     yield ('threads',) + names
 245     for c in concurrency:
 246         sess.concurrency = c
 247         sess.run()
 248         row = [c]
 249         for attr in attrs:
 250             val = getattr(sess, attr)
 251             if val is None:
 252                 print(sess.output)
 253                 row = None
 254                 break
 255             val = float(val)
 256             avg[attr] += float(val)
 257             row.append(val)
 258         if row:
 259             yield row
 260
 261     # Add a row of averages.
 262     yield ["Average"] + [str(avg[attr] / len(concurrency)) for attr in attrs]
 263
 264 def size_report(sizes=(10, 100, 1000, 10000, 100000, 100000000),
 265                concurrency=50):
 266     sess = ABSession(concurrency=concurrency)
 267     attrs, names, patterns = list(zip(*sess.parse_patterns))
 268     yield ('bytes',) + names
 269     for sz in sizes:
 270         sess.path = "%s/sizer?size=%s" % (SCRIPT_NAME, sz)
 271         sess.run()
 272         yield [sz] + [getattr(sess, attr) for attr in attrs]
 273
 274 def print_report(rows):
 275     for row in rows:
 276         print("")
 277         for i, val in enumerate(row):
 278             sys.stdout.write(str(val).rjust(10) + " | ")
 279     print("")
 280
 281
 282 def run_standard_benchmarks():
 283     print("")
 284     print("Client Thread Report (1000 requests, 14 byte response body, "
 285            "%s server threads):" % cherrypy.server.thread_pool)
 286     print_report(thread_report())
 287
 288     print("")
 289     print("Client Thread Report (1000 requests, 14 bytes via staticdir, "
 290            "%s server threads):" % cherrypy.server.thread_pool)
 291     print_report(thread_report("%s/static/index.html" % SCRIPT_NAME))
 292
 293     print("")
 294     print("Size Report (1000 requests, 50 client threads, "
 295            "%s server threads):" % cherrypy.server.thread_pool)
 296     print_report(size_report())
 297
 298
 299 #                         modpython and other WSGI                         #
 300
 301 def startup_modpython(req=None):
 302     """Start the CherryPy app server in 'serverless' mode (for modpython/WSGI)."""
 303     if cherrypy.engine.state == cherrypy._cpengine.STOPPED:
 304         if req:
 305             if "nullreq" in req.get_options():
 306                 cherrypy.engine.request_class = NullRequest
 307                 cherrypy.engine.response_class = NullResponse
 308             ab_opt = req.get_options().get("ab", "")
 309             if ab_opt:
 310                 global AB_PATH
 311                 AB_PATH = ab_opt
 312         cherrypy.engine.start()
 313     if cherrypy.engine.state == cherrypy._cpengine.STARTING:
 314         cherrypy.engine.wait()
 315     return 0 # apache.OK
 316
 317
 318 def run_modpython(use_wsgi=False):
 319     print("Starting mod_python...")
 320     pyopts = []
 321
 322     # Pass the null and ab=path options through Apache
 323     if "--null" in opts:
 324         pyopts.append(("nullreq", ""))
 325
 326     if "--ab" in opts:
 327         pyopts.append(("ab", opts["--ab"]))
 328
 329     s = _cpmodpy.ModPythonServer
 330     if use_wsgi:
 331         pyopts.append(("wsgi.application", "cherrypy::tree"))
 332         pyopts.append(("wsgi.startup", "cherrypy.test.benchmark::startup_modpython"))
 333         handler = "modpython_gateway::handler"
 334         s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH, handler=handler)
 335     else:
 336         pyopts.append(("cherrypy.setup", "cherrypy.test.benchmark::startup_modpython"))
 337         s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH)
 338
 339     try:
 340         s.start()
 341         run()
 342     finally:
 343         s.stop()
 344
 345
 346
 347 if __name__ == '__main__':
 348     longopts = ['cpmodpy', 'modpython', 'null', 'notests',
 349                 'help', 'ab=', 'apache=']
 350     try:
 351         switches, args = getopt.getopt(sys.argv[1:], "", longopts)
 352         opts = dict(switches)
 353     except getopt.GetoptError:
 354         print(__doc__)
 355         sys.exit(2)
 356
 357     if "--help" in opts:
 358         print(__doc__)
 359         sys.exit(0)
 360
 361     if "--ab" in opts:
 362         AB_PATH = opts['--ab']
 363
 364     if "--notests" in opts:
 365         # Return without stopping the server, so that the pages
 366         # can be tested from a standard web browser.
 367         def run():
 368             port = cherrypy.server.socket_port
 369             print("You may now open http://127.0.0.1:%s%s/" %
 370                    (port, SCRIPT_NAME))
 371
 372             if "--null" in opts:
 373                 print("Using null Request object")
 374     else:
 375         def run():
 376             end = time.time() - start
 377             print("Started in %s seconds" % end)
 378             if "--null" in opts:
 379                 print("\nUsing null Request object")
 380             try:
 381                 try:
 382                     run_standard_benchmarks()
 383                 except:
 384                     print(_cperror.format_exc())
 385                     raise
 386             finally:
 387                 cherrypy.engine.exit()
 388
 389     print("Starting CherryPy app server...")
 390
 391     class NullWriter(object):
 392         """Suppresses the printing of socket errors."""
 393         def write(self, data):
 394             pass
 395     sys.stderr = NullWriter()
 396
 397     start = time.time()
 398
 399     if "--cpmodpy" in opts:
 400         run_modpython()
 401     elif "--modpython" in opts:
 402         run_modpython(use_wsgi=True)
 403     else:
 404         if "--null" in opts:
 405             cherrypy.server.request_class = NullRequest
 406             cherrypy.server.response_class = NullResponse
 407
 408         cherrypy.engine.start_with_callback(run)
 409         cherrypy.engine.block()