Lib/pstats.py

   1 """Class for printing reports on profiled python code."""
   2
   3 # Class for printing reports on profiled python code. rev 1.0  4/1/94
   4 #
   5 # Based on prior profile module by Sjoerd Mullender...
   6 #   which was hacked somewhat by: Guido van Rossum
   7 #
   8 # see profile.doc and profile.py for more info.
   9
  10 # Copyright 1994, by InfoSeek Corporation, all rights reserved.
  11 # Written by James Roskind
  12 #
  13 # Permission to use, copy, modify, and distribute this Python software
  14 # and its associated documentation for any purpose (subject to the
  15 # restriction in the following sentence) without fee is hereby granted,
  16 # provided that the above copyright notice appears in all copies, and
  17 # that both that copyright notice and this permission notice appear in
  18 # supporting documentation, and that the name of InfoSeek not be used in
  19 # advertising or publicity pertaining to distribution of the software
  20 # without specific, written prior permission.  This permission is
  21 # explicitly restricted to the copying and modification of the software
  22 # to remain in Python, compiled Python, or other languages (such as C)
  23 # wherein the modified or derived code is exclusively imported into a
  24 # Python module.
  25 #
  26 # INFOSEEK CORPORATION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  27 # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  28 # FITNESS. IN NO EVENT SHALL INFOSEEK CORPORATION BE LIABLE FOR ANY
  29 # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  30 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
  31 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  32 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  33
  34
  35 import os
  36 import time
  37 import marshal
  38 import re
  39
  40 __all__ = ["Stats"]
  41
  42 class Stats:
  43     """This class is used for creating reports from data generated by the
  44     Profile class.  It is a "friend" of that class, and imports data either
  45     by direct access to members of Profile class, or by reading in a dictionary
  46     that was emitted (via marshal) from the Profile class.
  47
  48     The big change from the previous Profiler (in terms of raw functionality)
  49     is that an "add()" method has been provided to combine Stats from
  50     several distinct profile runs.  Both the constructor and the add()
  51     method now take arbitrarily many file names as arguments.
  52
  53     All the print methods now take an argument that indicates how many lines
  54     to print.  If the arg is a floating point number between 0 and 1.0, then
  55     it is taken as a decimal percentage of the available lines to be printed
  56     (e.g., .1 means print 10% of all available lines).  If it is an integer,
  57     it is taken to mean the number of lines of data that you wish to have
  58     printed.
  59
  60     The sort_stats() method now processes some additional options (i.e., in
  61     addition to the old -1, 0, 1, or 2).  It takes an arbitrary number of quoted
  62     strings to select the sort order.  For example sort_stats('time', 'name')
  63     sorts on the major key of "internal function time", and on the minor
  64     key of 'the name of the function'.  Look at the two tables in sort_stats()
  65     and get_sort_arg_defs(self) for more examples.
  66
  67     All methods now return "self",  so you can string together commands like:
  68         Stats('foo', 'goo').strip_dirs().sort_stats('calls').\
  69                             print_stats(5).print_callers(5)
  70     """
  71
  72     def __init__(self, *args):
  73         if not len(args):
  74             arg = None
  75         else:
  76             arg = args[0]
  77             args = args[1:]
  78         self.init(arg)
  79         self.add(*args)
  80
  81     def init(self, arg):
  82         self.all_callees = None  # calc only if needed
  83         self.files = []
  84         self.fcn_list = None
  85         self.total_tt = 0
  86         self.total_calls = 0
  87         self.prim_calls = 0
  88         self.max_name_len = 0
  89         self.top_level = {}
  90         self.stats = {}
  91         self.sort_arg_dict = {}
  92         self.load_stats(arg)
  93         trouble = 1
  94         try:
  95             self.get_top_level_stats()
  96             trouble = 0
  97         finally:
  98             if trouble:
  99                 print "Invalid timing data",
 100                 if self.files: print self.files[-1],
 101                 print
 102
 103     def load_stats(self, arg):
 104         if not arg:  self.stats = {}
 105         elif type(arg) == type(""):
 106             f = open(arg, 'rb')
 107             self.stats = marshal.load(f)
 108             f.close()
 109             try:
 110                 file_stats = os.stat(arg)
 111                 arg = time.ctime(file_stats.st_mtime) + "    " + arg
 112             except:  # in case this is not unix
 113                 pass
 114             self.files = [ arg ]
 115         elif hasattr(arg, 'create_stats'):
 116             arg.create_stats()
 117             self.stats = arg.stats
 118             arg.stats = {}
 119         if not self.stats:
 120             raise TypeError,  "Cannot create or construct a %r object from '%r''" % (
 121                               self.__class__, arg)
 122         return
 123
 124     def get_top_level_stats(self):
 125         for func, (cc, nc, tt, ct, callers) in self.stats.items():
 126             self.total_calls += nc
 127             self.prim_calls  += cc
 128             self.total_tt    += tt
 129             if callers.has_key(("jprofile", 0, "profiler")):
 130                 self.top_level[func] = None
 131             if len(func_std_string(func)) > self.max_name_len:
 132                 self.max_name_len = len(func_std_string(func))
 133
 134     def add(self, *arg_list):
 135         if not arg_list: return self
 136         if len(arg_list) > 1: self.add(*arg_list[1:])
 137         other = arg_list[0]
 138         if type(self) != type(other) or self.__class__ != other.__class__:
 139             other = Stats(other)
 140         self.files += other.files
 141         self.total_calls += other.total_calls
 142         self.prim_calls += other.prim_calls
 143         self.total_tt += other.total_tt
 144         for func in other.top_level:
 145             self.top_level[func] = None
 146
 147         if self.max_name_len < other.max_name_len:
 148             self.max_name_len = other.max_name_len
 149
 150         self.fcn_list = None
 151
 152         for func, stat in other.stats.iteritems():
 153             if func in self.stats:
 154                 old_func_stat = self.stats[func]
 155             else:
 156                 old_func_stat = (0, 0, 0, 0, {},)
 157             self.stats[func] = add_func_stats(old_func_stat, stat)
 158         return self
 159
 160     def dump_stats(self, filename):
 161         """Write the profile data to a file we know how to load back."""
 162         f = file(filename, 'wb')
 163         try:
 164             marshal.dump(self.stats, f)
 165         finally:
 166             f.close()
 167
 168     # list the tuple indices and directions for sorting,
 169     # along with some printable description
 170     sort_arg_dict_default = {
 171               "calls"     : (((1,-1),              ), "call count"),
 172               "cumulative": (((3,-1),              ), "cumulative time"),
 173               "file"      : (((4, 1),              ), "file name"),
 174               "line"      : (((5, 1),              ), "line number"),
 175               "module"    : (((4, 1),              ), "file name"),
 176               "name"      : (((6, 1),              ), "function name"),
 177               "nfl"       : (((6, 1),(4, 1),(5, 1),), "name/file/line"),
 178               "pcalls"    : (((0,-1),              ), "call count"),
 179               "stdname"   : (((7, 1),              ), "standard name"),
 180               "time"      : (((2,-1),              ), "internal time"),
 181               }
 182
 183     def get_sort_arg_defs(self):
 184         """Expand all abbreviations that are unique."""
 185         if not self.sort_arg_dict:
 186             self.sort_arg_dict = dict = {}
 187             bad_list = {}
 188             for word, tup in self.sort_arg_dict_default.iteritems():
 189                 fragment = word
 190                 while fragment:
 191                     if not fragment:
 192                         break
 193                     if fragment in dict:
 194                         bad_list[fragment] = 0
 195                         break
 196                     dict[fragment] = tup
 197                     fragment = fragment[:-1]
 198             for word in bad_list:
 199                 del dict[word]
 200         return self.sort_arg_dict
 201
 202     def sort_stats(self, *field):
 203         if not field:
 204             self.fcn_list = 0
 205             return self
 206         if len(field) == 1 and type(field[0]) == type(1):
 207             # Be compatible with old profiler
 208             field = [ {-1: "stdname",
 209                       0:"calls",
 210                       1:"time",
 211                       2: "cumulative" }  [ field[0] ] ]
 212
 213         sort_arg_defs = self.get_sort_arg_defs()
 214         sort_tuple = ()
 215         self.sort_type = ""
 216         connector = ""
 217         for word in field:
 218             sort_tuple = sort_tuple + sort_arg_defs[word][0]
 219             self.sort_type += connector + sort_arg_defs[word][1]
 220             connector = ", "
 221
 222         stats_list = []
 223         for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
 224             stats_list.append((cc, nc, tt, ct) + func +
 225                               (func_std_string(func), func))
 226
 227         stats_list.sort(TupleComp(sort_tuple).compare)
 228
 229         self.fcn_list = fcn_list = []
 230         for tuple in stats_list:
 231             fcn_list.append(tuple[-1])
 232         return self
 233
 234     def reverse_order(self):
 235         if self.fcn_list:
 236             self.fcn_list.reverse()
 237         return self
 238
 239     def strip_dirs(self):
 240         oldstats = self.stats
 241         self.stats = newstats = {}
 242         max_name_len = 0
 243         for func, (cc, nc, tt, ct, callers) in oldstats.iteritems():
 244             newfunc = func_strip_path(func)
 245             if len(func_std_string(newfunc)) > max_name_len:
 246                 max_name_len = len(func_std_string(newfunc))
 247             newcallers = {}
 248             for func2, caller in callers.iteritems():
 249                 newcallers[func_strip_path(func2)] = caller
 250
 251             if newfunc in newstats:
 252                 newstats[newfunc] = add_func_stats(
 253                                         newstats[newfunc],
 254                                         (cc, nc, tt, ct, newcallers))
 255             else:
 256                 newstats[newfunc] = (cc, nc, tt, ct, newcallers)
 257         old_top = self.top_level
 258         self.top_level = new_top = {}
 259         for func in old_top:
 260             new_top[func_strip_path(func)] = None
 261
 262         self.max_name_len = max_name_len
 263
 264         self.fcn_list = None
 265         self.all_callees = None
 266         return self
 267
 268     def calc_callees(self):
 269         if self.all_callees: return
 270         self.all_callees = all_callees = {}
 271         for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
 272             if not func in all_callees:
 273                 all_callees[func] = {}
 274             for func2, caller in callers.iteritems():
 275                 if not func2 in all_callees:
 276                     all_callees[func2] = {}
 277                 all_callees[func2][func]  = caller
 278         return
 279
 280     #******************************************************************
 281     # The following functions support actual printing of reports
 282     #******************************************************************
 283
 284     # Optional "amount" is either a line count, or a percentage of lines.
 285
 286     def eval_print_amount(self, sel, list, msg):
 287         new_list = list
 288         if type(sel) == type(""):
 289             new_list = []
 290             for func in list:
 291                 if re.search(sel, func_std_string(func)):
 292                     new_list.append(func)
 293         else:
 294             count = len(list)
 295             if type(sel) == type(1.0) and 0.0 <= sel < 1.0:
 296                 count = int(count * sel + .5)
 297                 new_list = list[:count]
 298             elif type(sel) == type(1) and 0 <= sel < count:
 299                 count = sel
 300                 new_list = list[:count]
 301         if len(list) != len(new_list):
 302             msg = msg + "   List reduced from %r to %r due to restriction <%r>\n" % (
 303                          len(list), len(new_list), sel)
 304
 305         return new_list, msg
 306
 307     def get_print_list(self, sel_list):
 308         width = self.max_name_len
 309         if self.fcn_list:
 310             list = self.fcn_list[:]
 311             msg = "   Ordered by: " + self.sort_type + '\n'
 312         else:
 313             list = self.stats.keys()
 314             msg = "   Random listing order was used\n"
 315
 316         for selection in sel_list:
 317             list, msg = self.eval_print_amount(selection, list, msg)
 318
 319         count = len(list)
 320
 321         if not list:
 322             return 0, list
 323         print msg
 324         if count < len(self.stats):
 325             width = 0
 326             for func in list:
 327                 if  len(func_std_string(func)) > width:
 328                     width = len(func_std_string(func))
 329         return width+2, list
 330
 331     def print_stats(self, *amount):
 332         for filename in self.files:
 333             print filename
 334         if self.files: print
 335         indent = ' ' * 8
 336         for func in self.top_level:
 337             print indent, func_get_function_name(func)
 338
 339         print indent, self.total_calls, "function calls",
 340         if self.total_calls != self.prim_calls:
 341             print "(%d primitive calls)" % self.prim_calls,
 342         print "in %.3f CPU seconds" % self.total_tt
 343         print
 344         width, list = self.get_print_list(amount)
 345         if list:
 346             self.print_title()
 347             for func in list:
 348                 self.print_line(func)
 349             print
 350             print
 351         return self
 352
 353     def print_callees(self, *amount):
 354         width, list = self.get_print_list(amount)
 355         if list:
 356             self.calc_callees()
 357
 358             self.print_call_heading(width, "called...")
 359             for func in list:
 360                 if func in self.all_callees:
 361                     self.print_call_line(width, func, self.all_callees[func])
 362                 else:
 363                     self.print_call_line(width, func, {})
 364             print
 365             print
 366         return self
 367
 368     def print_callers(self, *amount):
 369         width, list = self.get_print_list(amount)
 370         if list:
 371             self.print_call_heading(width, "was called by...")
 372             for func in list:
 373                 cc, nc, tt, ct, callers = self.stats[func]
 374                 self.print_call_line(width, func, callers, "<-")
 375             print
 376             print
 377         return self
 378
 379     def print_call_heading(self, name_size, column_title):
 380         print "Function ".ljust(name_size) + column_title
 381         # print sub-header only if we have new-style callers
 382         subheader = False
 383         for cc, nc, tt, ct, callers in self.stats.itervalues():
 384             if callers:
 385                 value = callers.itervalues().next()
 386                 subheader = isinstance(value, tuple)
 387                 break
 388         if subheader:
 389             print " "*name_size + "    ncalls  tottime  cumtime"
 390
 391     def print_call_line(self, name_size, source, call_dict, arrow="->"):
 392         print func_std_string(source).ljust(name_size) + arrow,
 393         if not call_dict:
 394             print
 395             return
 396         clist = call_dict.keys()
 397         clist.sort()
 398         indent = ""
 399         for func in clist:
 400             name = func_std_string(func)
 401             value = call_dict[func]
 402             if isinstance(value, tuple):
 403                 nc, cc, tt, ct = value
 404                 if nc != cc:
 405                     substats = '%d/%d' % (nc, cc)
 406                 else:
 407                     substats = '%d' % (nc,)
 408                 substats = '%s %s %s  %s' % (substats.rjust(7+2*len(indent)),
 409                                              f8(tt), f8(ct), name)
 410                 left_width = name_size + 1
 411             else:
 412                 substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3]))
 413                 left_width = name_size + 3
 414             print indent*left_width + substats
 415             indent = " "
 416
 417     def print_title(self):
 418         print '   ncalls  tottime  percall  cumtime  percall', \
 419               'filename:lineno(function)'
 420
 421     def print_line(self, func):  # hack : should print percentages
 422         cc, nc, tt, ct, callers = self.stats[func]
 423         c = str(nc)
 424         if nc != cc:
 425             c = c + '/' + str(cc)
 426         print c.rjust(9),
 427         print f8(tt),
 428         if nc == 0:
 429             print ' '*8,
 430         else:
 431             print f8(tt/nc),
 432         print f8(ct),
 433         if cc == 0:
 434             print ' '*8,
 435         else:
 436             print f8(ct/cc),
 437         print func_std_string(func)
 438
 439 class TupleComp:
 440     """This class provides a generic function for comparing any two tuples.
 441     Each instance records a list of tuple-indices (from most significant
 442     to least significant), and sort direction (ascending or decending) for
 443     each tuple-index.  The compare functions can then be used as the function
 444     argument to the system sort() function when a list of tuples need to be
 445     sorted in the instances order."""
 446
 447     def __init__(self, comp_select_list):
 448         self.comp_select_list = comp_select_list
 449
 450     def compare (self, left, right):
 451         for index, direction in self.comp_select_list:
 452             l = left[index]
 453             r = right[index]
 454             if l < r:
 455                 return -direction
 456             if l > r:
 457                 return direction
 458         return 0
 459
 460 #**************************************************************************
 461 # func_name is a triple (file:string, line:int, name:string)
 462
 463 def func_strip_path(func_name):
 464     filename, line, name = func_name
 465     return os.path.basename(filename), line, name
 466
 467 def func_get_function_name(func):
 468     return func[2]
 469
 470 def func_std_string(func_name): # match what old profile produced
 471     if func_name[:2] == ('~', 0):
 472         # special case for built-in functions
 473         name = func_name[2]
 474         if name.startswith('<') and name.endswith('>'):
 475             return '{%s}' % name[1:-1]
 476         else:
 477             return name
 478     else:
 479         return "%s:%d(%s)" % func_name
 480
 481 #**************************************************************************
 482 # The following functions combine statists for pairs functions.
 483 # The bulk of the processing involves correctly handling "call" lists,
 484 # such as callers and callees.
 485 #**************************************************************************
 486
 487 def add_func_stats(target, source):
 488     """Add together all the stats for two profile entries."""
 489     cc, nc, tt, ct, callers = source
 490     t_cc, t_nc, t_tt, t_ct, t_callers = target
 491     return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,
 492               add_callers(t_callers, callers))
 493
 494 def add_callers(target, source):
 495     """Combine two caller lists in a single list."""
 496     new_callers = {}
 497     for func, caller in target.iteritems():
 498         new_callers[func] = caller
 499     for func, caller in source.iteritems():
 500         if func in new_callers:
 501             new_callers[func] = caller + new_callers[func]
 502         else:
 503             new_callers[func] = caller
 504     return new_callers
 505
 506 def count_calls(callers):
 507     """Sum the caller statistics to get total number of calls received."""
 508     nc = 0
 509     for calls in callers.itervalues():
 510         nc += calls
 511     return nc
 512
 513 #**************************************************************************
 514 # The following functions support printing of reports
 515 #**************************************************************************
 516
 517 def f8(x):
 518     return "%8.3f" % x
 519
 520 #**************************************************************************
 521 # Statistics browser added by ESR, April 2001
 522 #**************************************************************************
 523
 524 if __name__ == '__main__':
 525     import cmd
 526     try:
 527         import readline
 528     except ImportError:
 529         pass
 530
 531     class ProfileBrowser(cmd.Cmd):
 532         def __init__(self, profile=None):
 533             cmd.Cmd.__init__(self)
 534             self.prompt = "% "
 535             if profile is not None:
 536                 self.stats = Stats(profile)
 537             else:
 538                 self.stats = None
 539
 540         def generic(self, fn, line):
 541             args = line.split()
 542             processed = []
 543             for term in args:
 544                 try:
 545                     processed.append(int(term))
 546                     continue
 547                 except ValueError:
 548                     pass
 549                 try:
 550                     frac = float(term)
 551                     if frac > 1 or frac < 0:
 552                         print "Fraction argument mus be in [0, 1]"
 553                         continue
 554                     processed.append(frac)
 555                     continue
 556                 except ValueError:
 557                     pass
 558                 processed.append(term)
 559             if self.stats:
 560                 getattr(self.stats, fn)(*processed)
 561             else:
 562                 print "No statistics object is loaded."
 563             return 0
 564         def generic_help(self):
 565             print "Arguments may be:"
 566             print "* An integer maximum number of entries to print."
 567             print "* A decimal fractional number between 0 and 1, controlling"
 568             print "  what fraction of selected entries to print."
 569             print "* A regular expression; only entries with function names"
 570             print "  that match it are printed."
 571
 572         def do_add(self, line):
 573             self.stats.add(line)
 574             return 0
 575         def help_add(self):
 576             print "Add profile info from given file to current statistics object."
 577
 578         def do_callees(self, line):
 579             return self.generic('print_callees', line)
 580         def help_callees(self):
 581             print "Print callees statistics from the current stat object."
 582             self.generic_help()
 583
 584         def do_callers(self, line):
 585             return self.generic('print_callers', line)
 586         def help_callers(self):
 587             print "Print callers statistics from the current stat object."
 588             self.generic_help()
 589
 590         def do_EOF(self, line):
 591             print ""
 592             return 1
 593         def help_EOF(self):
 594             print "Leave the profile brower."
 595
 596         def do_quit(self, line):
 597             return 1
 598         def help_quit(self):
 599             print "Leave the profile brower."
 600
 601         def do_read(self, line):
 602             if line:
 603                 try:
 604                     self.stats = Stats(line)
 605                 except IOError, args:
 606                     print args[1]
 607                     return
 608                 self.prompt = line + "% "
 609             elif len(self.prompt) > 2:
 610                 line = self.prompt[-2:]
 611             else:
 612                 print "No statistics object is current -- cannot reload."
 613             return 0
 614         def help_read(self):
 615             print "Read in profile data from a specified file."
 616
 617         def do_reverse(self, line):
 618             self.stats.reverse_order()
 619             return 0
 620         def help_reverse(self):
 621             print "Reverse the sort order of the profiling report."
 622
 623         def do_sort(self, line):
 624             abbrevs = self.stats.get_sort_arg_defs()
 625             if line and not filter(lambda x,a=abbrevs: x not in a,line.split()):
 626                 self.stats.sort_stats(*line.split())
 627             else:
 628                 print "Valid sort keys (unique prefixes are accepted):"
 629                 for (key, value) in Stats.sort_arg_dict_default.iteritems():
 630                     print "%s -- %s" % (key, value[1])
 631             return 0
 632         def help_sort(self):
 633             print "Sort profile data according to specified keys."
 634             print "(Typing `sort' without arguments lists valid keys.)"
 635         def complete_sort(self, text, *args):
 636             return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]
 637
 638         def do_stats(self, line):
 639             return self.generic('print_stats', line)
 640         def help_stats(self):
 641             print "Print statistics from the current stat object."
 642             self.generic_help()
 643
 644         def do_strip(self, line):
 645             self.stats.strip_dirs()
 646             return 0
 647         def help_strip(self):
 648             print "Strip leading path information from filenames in the report."
 649
 650         def postcmd(self, stop, line):
 651             if stop:
 652                 return stop
 653             return None
 654
 655     import sys
 656     print "Welcome to the profile statistics browser."
 657     if len(sys.argv) > 1:
 658         initprofile = sys.argv[1]
 659     else:
 660         initprofile = None
 661     try:
 662         ProfileBrowser(initprofile).cmdloop()
 663         print "Goodbye."
 664     except KeyboardInterrupt:
 665         pass
 666
 667 # That's all, folks.