Remove the regrtest check that turns any ImportError into a skipped test.
[python.git] / Lib / pstats.py
bloba6844fbe9444637320fba3b55a9b4a3cf9f39388
1 """Class for printing reports on profiled python code."""
3 # Class for printing reports on profiled python code. rev 1.0 4/1/94
5 # Based on prior profile module by Sjoerd Mullender...
6 # which was hacked somewhat by: Guido van Rossum
8 # see profile.doc and profile.py for more info.
10 # Copyright 1994, by InfoSeek Corporation, all rights reserved.
11 # Written by James Roskind
13 # Permission to use, copy, modify, and distribute this Python software
14 # and its associated documentation for any purpose (subject to the
15 # restriction in the following sentence) without fee is hereby granted,
16 # provided that the above copyright notice appears in all copies, and
17 # that both that copyright notice and this permission notice appear in
18 # supporting documentation, and that the name of InfoSeek not be used in
19 # advertising or publicity pertaining to distribution of the software
20 # without specific, written prior permission. This permission is
21 # explicitly restricted to the copying and modification of the software
22 # to remain in Python, compiled Python, or other languages (such as C)
23 # wherein the modified or derived code is exclusively imported into a
24 # Python module.
26 # INFOSEEK CORPORATION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
27 # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
28 # FITNESS. IN NO EVENT SHALL INFOSEEK CORPORATION BE LIABLE FOR ANY
29 # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
30 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
31 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
32 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
35 import sys
36 import os
37 import time
38 import marshal
39 import re
41 __all__ = ["Stats"]
43 class Stats:
44 """This class is used for creating reports from data generated by the
45 Profile class. It is a "friend" of that class, and imports data either
46 by direct access to members of Profile class, or by reading in a dictionary
47 that was emitted (via marshal) from the Profile class.
49 The big change from the previous Profiler (in terms of raw functionality)
50 is that an "add()" method has been provided to combine Stats from
51 several distinct profile runs. Both the constructor and the add()
52 method now take arbitrarily many file names as arguments.
54 All the print methods now take an argument that indicates how many lines
55 to print. If the arg is a floating point number between 0 and 1.0, then
56 it is taken as a decimal percentage of the available lines to be printed
57 (e.g., .1 means print 10% of all available lines). If it is an integer,
58 it is taken to mean the number of lines of data that you wish to have
59 printed.
61 The sort_stats() method now processes some additional options (i.e., in
62 addition to the old -1, 0, 1, or 2). It takes an arbitrary number of
63 quoted strings to select the sort order. For example sort_stats('time',
64 'name') sorts on the major key of 'internal function time', and on the
65 minor key of 'the name of the function'. Look at the two tables in
66 sort_stats() and get_sort_arg_defs(self) for more examples.
68 All methods return self, so you can string together commands like:
69 Stats('foo', 'goo').strip_dirs().sort_stats('calls').\
70 print_stats(5).print_callers(5)
71 """
73 def __init__(self, *args, **kwds):
74 # I can't figure out how to explictly specify a stream keyword arg
75 # with *args:
76 # def __init__(self, *args, stream=sys.stdout): ...
77 # so I use **kwds and sqauwk if something unexpected is passed in.
78 self.stream = sys.stdout
79 if "stream" in kwds:
80 self.stream = kwds["stream"]
81 del kwds["stream"]
82 if kwds:
83 keys = kwds.keys()
84 keys.sort()
85 extras = ", ".join(["%s=%s" % (k, kwds[k]) for k in keys])
86 raise ValueError, "unrecognized keyword args: %s" % extras
87 if not len(args):
88 arg = None
89 else:
90 arg = args[0]
91 args = args[1:]
92 self.init(arg)
93 self.add(*args)
95 def init(self, arg):
96 self.all_callees = None # calc only if needed
97 self.files = []
98 self.fcn_list = None
99 self.total_tt = 0
100 self.total_calls = 0
101 self.prim_calls = 0
102 self.max_name_len = 0
103 self.top_level = {}
104 self.stats = {}
105 self.sort_arg_dict = {}
106 self.load_stats(arg)
107 trouble = 1
108 try:
109 self.get_top_level_stats()
110 trouble = 0
111 finally:
112 if trouble:
113 print >> self.stream, "Invalid timing data",
114 if self.files: print >> self.stream, self.files[-1],
115 print >> self.stream
117 def load_stats(self, arg):
118 if not arg: self.stats = {}
119 elif isinstance(arg, basestring):
120 f = open(arg, 'rb')
121 self.stats = marshal.load(f)
122 f.close()
123 try:
124 file_stats = os.stat(arg)
125 arg = time.ctime(file_stats.st_mtime) + " " + arg
126 except: # in case this is not unix
127 pass
128 self.files = [ arg ]
129 elif hasattr(arg, 'create_stats'):
130 arg.create_stats()
131 self.stats = arg.stats
132 arg.stats = {}
133 if not self.stats:
134 raise TypeError, "Cannot create or construct a %r object from '%r''" % (
135 self.__class__, arg)
136 return
138 def get_top_level_stats(self):
139 for func, (cc, nc, tt, ct, callers) in self.stats.items():
140 self.total_calls += nc
141 self.prim_calls += cc
142 self.total_tt += tt
143 if ("jprofile", 0, "profiler") in callers:
144 self.top_level[func] = None
145 if len(func_std_string(func)) > self.max_name_len:
146 self.max_name_len = len(func_std_string(func))
148 def add(self, *arg_list):
149 if not arg_list: return self
150 if len(arg_list) > 1: self.add(*arg_list[1:])
151 other = arg_list[0]
152 if type(self) != type(other) or self.__class__ != other.__class__:
153 other = Stats(other)
154 self.files += other.files
155 self.total_calls += other.total_calls
156 self.prim_calls += other.prim_calls
157 self.total_tt += other.total_tt
158 for func in other.top_level:
159 self.top_level[func] = None
161 if self.max_name_len < other.max_name_len:
162 self.max_name_len = other.max_name_len
164 self.fcn_list = None
166 for func, stat in other.stats.iteritems():
167 if func in self.stats:
168 old_func_stat = self.stats[func]
169 else:
170 old_func_stat = (0, 0, 0, 0, {},)
171 self.stats[func] = add_func_stats(old_func_stat, stat)
172 return self
174 def dump_stats(self, filename):
175 """Write the profile data to a file we know how to load back."""
176 f = file(filename, 'wb')
177 try:
178 marshal.dump(self.stats, f)
179 finally:
180 f.close()
182 # list the tuple indices and directions for sorting,
183 # along with some printable description
184 sort_arg_dict_default = {
185 "calls" : (((1,-1), ), "call count"),
186 "cumulative": (((3,-1), ), "cumulative time"),
187 "file" : (((4, 1), ), "file name"),
188 "line" : (((5, 1), ), "line number"),
189 "module" : (((4, 1), ), "file name"),
190 "name" : (((6, 1), ), "function name"),
191 "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"),
192 "pcalls" : (((0,-1), ), "call count"),
193 "stdname" : (((7, 1), ), "standard name"),
194 "time" : (((2,-1), ), "internal time"),
197 def get_sort_arg_defs(self):
198 """Expand all abbreviations that are unique."""
199 if not self.sort_arg_dict:
200 self.sort_arg_dict = dict = {}
201 bad_list = {}
202 for word, tup in self.sort_arg_dict_default.iteritems():
203 fragment = word
204 while fragment:
205 if not fragment:
206 break
207 if fragment in dict:
208 bad_list[fragment] = 0
209 break
210 dict[fragment] = tup
211 fragment = fragment[:-1]
212 for word in bad_list:
213 del dict[word]
214 return self.sort_arg_dict
216 def sort_stats(self, *field):
217 if not field:
218 self.fcn_list = 0
219 return self
220 if len(field) == 1 and type(field[0]) == type(1):
221 # Be compatible with old profiler
222 field = [ {-1: "stdname",
223 0:"calls",
224 1:"time",
225 2: "cumulative" } [ field[0] ] ]
227 sort_arg_defs = self.get_sort_arg_defs()
228 sort_tuple = ()
229 self.sort_type = ""
230 connector = ""
231 for word in field:
232 sort_tuple = sort_tuple + sort_arg_defs[word][0]
233 self.sort_type += connector + sort_arg_defs[word][1]
234 connector = ", "
236 stats_list = []
237 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
238 stats_list.append((cc, nc, tt, ct) + func +
239 (func_std_string(func), func))
241 stats_list.sort(key=CmpToKey(TupleComp(sort_tuple).compare))
243 self.fcn_list = fcn_list = []
244 for tuple in stats_list:
245 fcn_list.append(tuple[-1])
246 return self
248 def reverse_order(self):
249 if self.fcn_list:
250 self.fcn_list.reverse()
251 return self
253 def strip_dirs(self):
254 oldstats = self.stats
255 self.stats = newstats = {}
256 max_name_len = 0
257 for func, (cc, nc, tt, ct, callers) in oldstats.iteritems():
258 newfunc = func_strip_path(func)
259 if len(func_std_string(newfunc)) > max_name_len:
260 max_name_len = len(func_std_string(newfunc))
261 newcallers = {}
262 for func2, caller in callers.iteritems():
263 newcallers[func_strip_path(func2)] = caller
265 if newfunc in newstats:
266 newstats[newfunc] = add_func_stats(
267 newstats[newfunc],
268 (cc, nc, tt, ct, newcallers))
269 else:
270 newstats[newfunc] = (cc, nc, tt, ct, newcallers)
271 old_top = self.top_level
272 self.top_level = new_top = {}
273 for func in old_top:
274 new_top[func_strip_path(func)] = None
276 self.max_name_len = max_name_len
278 self.fcn_list = None
279 self.all_callees = None
280 return self
282 def calc_callees(self):
283 if self.all_callees: return
284 self.all_callees = all_callees = {}
285 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
286 if not func in all_callees:
287 all_callees[func] = {}
288 for func2, caller in callers.iteritems():
289 if not func2 in all_callees:
290 all_callees[func2] = {}
291 all_callees[func2][func] = caller
292 return
294 #******************************************************************
295 # The following functions support actual printing of reports
296 #******************************************************************
298 # Optional "amount" is either a line count, or a percentage of lines.
300 def eval_print_amount(self, sel, list, msg):
301 new_list = list
302 if type(sel) == type(""):
303 new_list = []
304 for func in list:
305 if re.search(sel, func_std_string(func)):
306 new_list.append(func)
307 else:
308 count = len(list)
309 if type(sel) == type(1.0) and 0.0 <= sel < 1.0:
310 count = int(count * sel + .5)
311 new_list = list[:count]
312 elif type(sel) == type(1) and 0 <= sel < count:
313 count = sel
314 new_list = list[:count]
315 if len(list) != len(new_list):
316 msg = msg + " List reduced from %r to %r due to restriction <%r>\n" % (
317 len(list), len(new_list), sel)
319 return new_list, msg
321 def get_print_list(self, sel_list):
322 width = self.max_name_len
323 if self.fcn_list:
324 list = self.fcn_list[:]
325 msg = " Ordered by: " + self.sort_type + '\n'
326 else:
327 list = self.stats.keys()
328 msg = " Random listing order was used\n"
330 for selection in sel_list:
331 list, msg = self.eval_print_amount(selection, list, msg)
333 count = len(list)
335 if not list:
336 return 0, list
337 print >> self.stream, msg
338 if count < len(self.stats):
339 width = 0
340 for func in list:
341 if len(func_std_string(func)) > width:
342 width = len(func_std_string(func))
343 return width+2, list
345 def print_stats(self, *amount):
346 for filename in self.files:
347 print >> self.stream, filename
348 if self.files: print >> self.stream
349 indent = ' ' * 8
350 for func in self.top_level:
351 print >> self.stream, indent, func_get_function_name(func)
353 print >> self.stream, indent, self.total_calls, "function calls",
354 if self.total_calls != self.prim_calls:
355 print >> self.stream, "(%d primitive calls)" % self.prim_calls,
356 print >> self.stream, "in %.3f CPU seconds" % self.total_tt
357 print >> self.stream
358 width, list = self.get_print_list(amount)
359 if list:
360 self.print_title()
361 for func in list:
362 self.print_line(func)
363 print >> self.stream
364 print >> self.stream
365 return self
367 def print_callees(self, *amount):
368 width, list = self.get_print_list(amount)
369 if list:
370 self.calc_callees()
372 self.print_call_heading(width, "called...")
373 for func in list:
374 if func in self.all_callees:
375 self.print_call_line(width, func, self.all_callees[func])
376 else:
377 self.print_call_line(width, func, {})
378 print >> self.stream
379 print >> self.stream
380 return self
382 def print_callers(self, *amount):
383 width, list = self.get_print_list(amount)
384 if list:
385 self.print_call_heading(width, "was called by...")
386 for func in list:
387 cc, nc, tt, ct, callers = self.stats[func]
388 self.print_call_line(width, func, callers, "<-")
389 print >> self.stream
390 print >> self.stream
391 return self
393 def print_call_heading(self, name_size, column_title):
394 print >> self.stream, "Function ".ljust(name_size) + column_title
395 # print sub-header only if we have new-style callers
396 subheader = False
397 for cc, nc, tt, ct, callers in self.stats.itervalues():
398 if callers:
399 value = callers.itervalues().next()
400 subheader = isinstance(value, tuple)
401 break
402 if subheader:
403 print >> self.stream, " "*name_size + " ncalls tottime cumtime"
405 def print_call_line(self, name_size, source, call_dict, arrow="->"):
406 print >> self.stream, func_std_string(source).ljust(name_size) + arrow,
407 if not call_dict:
408 print >> self.stream
409 return
410 clist = call_dict.keys()
411 clist.sort()
412 indent = ""
413 for func in clist:
414 name = func_std_string(func)
415 value = call_dict[func]
416 if isinstance(value, tuple):
417 nc, cc, tt, ct = value
418 if nc != cc:
419 substats = '%d/%d' % (nc, cc)
420 else:
421 substats = '%d' % (nc,)
422 substats = '%s %s %s %s' % (substats.rjust(7+2*len(indent)),
423 f8(tt), f8(ct), name)
424 left_width = name_size + 1
425 else:
426 substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3]))
427 left_width = name_size + 3
428 print >> self.stream, indent*left_width + substats
429 indent = " "
431 def print_title(self):
432 print >> self.stream, ' ncalls tottime percall cumtime percall',
433 print >> self.stream, 'filename:lineno(function)'
435 def print_line(self, func): # hack : should print percentages
436 cc, nc, tt, ct, callers = self.stats[func]
437 c = str(nc)
438 if nc != cc:
439 c = c + '/' + str(cc)
440 print >> self.stream, c.rjust(9),
441 print >> self.stream, f8(tt),
442 if nc == 0:
443 print >> self.stream, ' '*8,
444 else:
445 print >> self.stream, f8(tt/nc),
446 print >> self.stream, f8(ct),
447 if cc == 0:
448 print >> self.stream, ' '*8,
449 else:
450 print >> self.stream, f8(ct/cc),
451 print >> self.stream, func_std_string(func)
453 class TupleComp:
454 """This class provides a generic function for comparing any two tuples.
455 Each instance records a list of tuple-indices (from most significant
456 to least significant), and sort direction (ascending or decending) for
457 each tuple-index. The compare functions can then be used as the function
458 argument to the system sort() function when a list of tuples need to be
459 sorted in the instances order."""
461 def __init__(self, comp_select_list):
462 self.comp_select_list = comp_select_list
464 def compare (self, left, right):
465 for index, direction in self.comp_select_list:
466 l = left[index]
467 r = right[index]
468 if l < r:
469 return -direction
470 if l > r:
471 return direction
472 return 0
474 def CmpToKey(mycmp):
475 """Convert a cmp= function into a key= function"""
476 class K(object):
477 def __init__(self, obj):
478 self.obj = obj
479 def __lt__(self, other):
480 return mycmp(self.obj, other.obj) == -1
481 return K
484 #**************************************************************************
485 # func_name is a triple (file:string, line:int, name:string)
487 def func_strip_path(func_name):
488 filename, line, name = func_name
489 return os.path.basename(filename), line, name
491 def func_get_function_name(func):
492 return func[2]
494 def func_std_string(func_name): # match what old profile produced
495 if func_name[:2] == ('~', 0):
496 # special case for built-in functions
497 name = func_name[2]
498 if name.startswith('<') and name.endswith('>'):
499 return '{%s}' % name[1:-1]
500 else:
501 return name
502 else:
503 return "%s:%d(%s)" % func_name
505 #**************************************************************************
506 # The following functions combine statists for pairs functions.
507 # The bulk of the processing involves correctly handling "call" lists,
508 # such as callers and callees.
509 #**************************************************************************
511 def add_func_stats(target, source):
512 """Add together all the stats for two profile entries."""
513 cc, nc, tt, ct, callers = source
514 t_cc, t_nc, t_tt, t_ct, t_callers = target
515 return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,
516 add_callers(t_callers, callers))
518 def add_callers(target, source):
519 """Combine two caller lists in a single list."""
520 new_callers = {}
521 for func, caller in target.iteritems():
522 new_callers[func] = caller
523 for func, caller in source.iteritems():
524 if func in new_callers:
525 new_callers[func] = tuple([i[0] + i[1] for i in
526 zip(caller, new_callers[func])])
527 else:
528 new_callers[func] = caller
529 return new_callers
531 def count_calls(callers):
532 """Sum the caller statistics to get total number of calls received."""
533 nc = 0
534 for calls in callers.itervalues():
535 nc += calls
536 return nc
538 #**************************************************************************
539 # The following functions support printing of reports
540 #**************************************************************************
542 def f8(x):
543 return "%8.3f" % x
545 #**************************************************************************
546 # Statistics browser added by ESR, April 2001
547 #**************************************************************************
549 if __name__ == '__main__':
550 import cmd
551 try:
552 import readline
553 except ImportError:
554 pass
556 class ProfileBrowser(cmd.Cmd):
557 def __init__(self, profile=None):
558 cmd.Cmd.__init__(self)
559 self.prompt = "% "
560 if profile is not None:
561 self.stats = Stats(profile)
562 self.stream = self.stats.stream
563 else:
564 self.stats = None
565 self.stream = sys.stdout
567 def generic(self, fn, line):
568 args = line.split()
569 processed = []
570 for term in args:
571 try:
572 processed.append(int(term))
573 continue
574 except ValueError:
575 pass
576 try:
577 frac = float(term)
578 if frac > 1 or frac < 0:
579 print >> self.stream, "Fraction argument must be in [0, 1]"
580 continue
581 processed.append(frac)
582 continue
583 except ValueError:
584 pass
585 processed.append(term)
586 if self.stats:
587 getattr(self.stats, fn)(*processed)
588 else:
589 print >> self.stream, "No statistics object is loaded."
590 return 0
591 def generic_help(self):
592 print >> self.stream, "Arguments may be:"
593 print >> self.stream, "* An integer maximum number of entries to print."
594 print >> self.stream, "* A decimal fractional number between 0 and 1, controlling"
595 print >> self.stream, " what fraction of selected entries to print."
596 print >> self.stream, "* A regular expression; only entries with function names"
597 print >> self.stream, " that match it are printed."
599 def do_add(self, line):
600 self.stats.add(line)
601 return 0
602 def help_add(self):
603 print >> self.stream, "Add profile info from given file to current statistics object."
605 def do_callees(self, line):
606 return self.generic('print_callees', line)
607 def help_callees(self):
608 print >> self.stream, "Print callees statistics from the current stat object."
609 self.generic_help()
611 def do_callers(self, line):
612 return self.generic('print_callers', line)
613 def help_callers(self):
614 print >> self.stream, "Print callers statistics from the current stat object."
615 self.generic_help()
617 def do_EOF(self, line):
618 print >> self.stream, ""
619 return 1
620 def help_EOF(self):
621 print >> self.stream, "Leave the profile brower."
623 def do_quit(self, line):
624 return 1
625 def help_quit(self):
626 print >> self.stream, "Leave the profile brower."
628 def do_read(self, line):
629 if line:
630 try:
631 self.stats = Stats(line)
632 except IOError, args:
633 print >> self.stream, args[1]
634 return
635 self.prompt = line + "% "
636 elif len(self.prompt) > 2:
637 line = self.prompt[-2:]
638 else:
639 print >> self.stream, "No statistics object is current -- cannot reload."
640 return 0
641 def help_read(self):
642 print >> self.stream, "Read in profile data from a specified file."
644 def do_reverse(self, line):
645 self.stats.reverse_order()
646 return 0
647 def help_reverse(self):
648 print >> self.stream, "Reverse the sort order of the profiling report."
650 def do_sort(self, line):
651 abbrevs = self.stats.get_sort_arg_defs()
652 if line and not filter(lambda x,a=abbrevs: x not in a,line.split()):
653 self.stats.sort_stats(*line.split())
654 else:
655 print >> self.stream, "Valid sort keys (unique prefixes are accepted):"
656 for (key, value) in Stats.sort_arg_dict_default.iteritems():
657 print >> self.stream, "%s -- %s" % (key, value[1])
658 return 0
659 def help_sort(self):
660 print >> self.stream, "Sort profile data according to specified keys."
661 print >> self.stream, "(Typing `sort' without arguments lists valid keys.)"
662 def complete_sort(self, text, *args):
663 return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]
665 def do_stats(self, line):
666 return self.generic('print_stats', line)
667 def help_stats(self):
668 print >> self.stream, "Print statistics from the current stat object."
669 self.generic_help()
671 def do_strip(self, line):
672 self.stats.strip_dirs()
673 return 0
674 def help_strip(self):
675 print >> self.stream, "Strip leading path information from filenames in the report."
677 def postcmd(self, stop, line):
678 if stop:
679 return stop
680 return None
682 import sys
683 if len(sys.argv) > 1:
684 initprofile = sys.argv[1]
685 else:
686 initprofile = None
687 try:
688 browser = ProfileBrowser(initprofile)
689 print >> browser.stream, "Welcome to the profile statistics browser."
690 browser.cmdloop()
691 print >> browser.stream, "Goodbye."
692 except KeyboardInterrupt:
693 pass
695 # That's all, folks.