Add missing quotes.
[python.git] / Lib / doctest.py
blobb07ef6e4f35bcb24d52ca099d0e27ad75dacd02a
1 # Module doctest.
2 # Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org).
3 # Major enhancements and refactoring by:
4 # Jim Fulton
5 # Edward Loper
7 # Provided as-is; use at your own risk; no warranty; no promises; enjoy!
9 r"""Module doctest -- a framework for running examples in docstrings.
11 In simplest use, end each module M to be tested with:
13 def _test():
14 import doctest
15 doctest.testmod()
17 if __name__ == "__main__":
18 _test()
20 Then running the module as a script will cause the examples in the
21 docstrings to get executed and verified:
23 python M.py
25 This won't display anything unless an example fails, in which case the
26 failing example(s) and the cause(s) of the failure(s) are printed to stdout
27 (why not stderr? because stderr is a lame hack <0.2 wink>), and the final
28 line of output is "Test failed.".
30 Run it with the -v switch instead:
32 python M.py -v
34 and a detailed report of all examples tried is printed to stdout, along
35 with assorted summaries at the end.
37 You can force verbose mode by passing "verbose=True" to testmod, or prohibit
38 it by passing "verbose=False". In either of those cases, sys.argv is not
39 examined by testmod.
41 There are a variety of other ways to run doctests, including integration
42 with the unittest framework, and support for running non-Python text
43 files containing doctests. There are also many ways to override parts
44 of doctest's default behaviors. See the Library Reference Manual for
45 details.
46 """
48 __docformat__ = 'reStructuredText en'
50 __all__ = [
51 # 0, Option Flags
52 'register_optionflag',
53 'DONT_ACCEPT_TRUE_FOR_1',
54 'DONT_ACCEPT_BLANKLINE',
55 'NORMALIZE_WHITESPACE',
56 'ELLIPSIS',
57 'SKIP',
58 'IGNORE_EXCEPTION_DETAIL',
59 'COMPARISON_FLAGS',
60 'REPORT_UDIFF',
61 'REPORT_CDIFF',
62 'REPORT_NDIFF',
63 'REPORT_ONLY_FIRST_FAILURE',
64 'REPORTING_FLAGS',
65 # 1. Utility Functions
66 # 2. Example & DocTest
67 'Example',
68 'DocTest',
69 # 3. Doctest Parser
70 'DocTestParser',
71 # 4. Doctest Finder
72 'DocTestFinder',
73 # 5. Doctest Runner
74 'DocTestRunner',
75 'OutputChecker',
76 'DocTestFailure',
77 'UnexpectedException',
78 'DebugRunner',
79 # 6. Test Functions
80 'testmod',
81 'testfile',
82 'run_docstring_examples',
83 # 7. Tester
84 'Tester',
85 # 8. Unittest Support
86 'DocTestSuite',
87 'DocFileSuite',
88 'set_unittest_reportflags',
89 # 9. Debugging Support
90 'script_from_examples',
91 'testsource',
92 'debug_src',
93 'debug',
96 import __future__
98 import sys, traceback, inspect, linecache, os, re
99 import unittest, difflib, pdb, tempfile
100 import warnings
101 from StringIO import StringIO
102 from collections import namedtuple
104 TestResults = namedtuple('TestResults', 'failed attempted')
106 # There are 4 basic classes:
107 # - Example: a <source, want> pair, plus an intra-docstring line number.
108 # - DocTest: a collection of examples, parsed from a docstring, plus
109 # info about where the docstring came from (name, filename, lineno).
110 # - DocTestFinder: extracts DocTests from a given object's docstring and
111 # its contained objects' docstrings.
112 # - DocTestRunner: runs DocTest cases, and accumulates statistics.
114 # So the basic picture is:
116 # list of:
117 # +------+ +---------+ +-------+
118 # |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results|
119 # +------+ +---------+ +-------+
120 # | Example |
121 # | ... |
122 # | Example |
123 # +---------+
125 # Option constants.
127 OPTIONFLAGS_BY_NAME = {}
128 def register_optionflag(name):
129 # Create a new flag unless `name` is already known.
130 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME))
132 DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1')
133 DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE')
134 NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE')
135 ELLIPSIS = register_optionflag('ELLIPSIS')
136 SKIP = register_optionflag('SKIP')
137 IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL')
139 COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 |
140 DONT_ACCEPT_BLANKLINE |
141 NORMALIZE_WHITESPACE |
142 ELLIPSIS |
143 SKIP |
144 IGNORE_EXCEPTION_DETAIL)
146 REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
147 REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
148 REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
149 REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
151 REPORTING_FLAGS = (REPORT_UDIFF |
152 REPORT_CDIFF |
153 REPORT_NDIFF |
154 REPORT_ONLY_FIRST_FAILURE)
156 # Special string markers for use in `want` strings:
157 BLANKLINE_MARKER = '<BLANKLINE>'
158 ELLIPSIS_MARKER = '...'
160 ######################################################################
161 ## Table of Contents
162 ######################################################################
163 # 1. Utility Functions
164 # 2. Example & DocTest -- store test cases
165 # 3. DocTest Parser -- extracts examples from strings
166 # 4. DocTest Finder -- extracts test cases from objects
167 # 5. DocTest Runner -- runs test cases
168 # 6. Test Functions -- convenient wrappers for testing
169 # 7. Tester Class -- for backwards compatibility
170 # 8. Unittest Support
171 # 9. Debugging Support
172 # 10. Example Usage
174 ######################################################################
175 ## 1. Utility Functions
176 ######################################################################
178 def _extract_future_flags(globs):
180 Return the compiler-flags associated with the future features that
181 have been imported into the given namespace (globs).
183 flags = 0
184 for fname in __future__.all_feature_names:
185 feature = globs.get(fname, None)
186 if feature is getattr(__future__, fname):
187 flags |= feature.compiler_flag
188 return flags
190 def _normalize_module(module, depth=2):
192 Return the module specified by `module`. In particular:
193 - If `module` is a module, then return module.
194 - If `module` is a string, then import and return the
195 module with that name.
196 - If `module` is None, then return the calling module.
197 The calling module is assumed to be the module of
198 the stack frame at the given depth in the call stack.
200 if inspect.ismodule(module):
201 return module
202 elif isinstance(module, (str, unicode)):
203 return __import__(module, globals(), locals(), ["*"])
204 elif module is None:
205 return sys.modules[sys._getframe(depth).f_globals['__name__']]
206 else:
207 raise TypeError("Expected a module, string, or None")
209 def _load_testfile(filename, package, module_relative):
210 if module_relative:
211 package = _normalize_module(package, 3)
212 filename = _module_relative_path(package, filename)
213 if hasattr(package, '__loader__'):
214 if hasattr(package.__loader__, 'get_data'):
215 file_contents = package.__loader__.get_data(filename)
216 # get_data() opens files as 'rb', so one must do the equivalent
217 # conversion as universal newlines would do.
218 return file_contents.replace(os.linesep, '\n'), filename
219 return open(filename).read(), filename
221 def _indent(s, indent=4):
223 Add the given number of space characters to the beginning every
224 non-blank line in `s`, and return the result.
226 # This regexp matches the start of non-blank lines:
227 return re.sub('(?m)^(?!$)', indent*' ', s)
229 def _exception_traceback(exc_info):
231 Return a string containing a traceback message for the given
232 exc_info tuple (as returned by sys.exc_info()).
234 # Get a traceback message.
235 excout = StringIO()
236 exc_type, exc_val, exc_tb = exc_info
237 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)
238 return excout.getvalue()
240 # Override some StringIO methods.
241 class _SpoofOut(StringIO):
242 def getvalue(self):
243 result = StringIO.getvalue(self)
244 # If anything at all was written, make sure there's a trailing
245 # newline. There's no way for the expected output to indicate
246 # that a trailing newline is missing.
247 if result and not result.endswith("\n"):
248 result += "\n"
249 # Prevent softspace from screwing up the next test case, in
250 # case they used print with a trailing comma in an example.
251 if hasattr(self, "softspace"):
252 del self.softspace
253 return result
255 def truncate(self, size=None):
256 StringIO.truncate(self, size)
257 if hasattr(self, "softspace"):
258 del self.softspace
260 # Worst-case linear-time ellipsis matching.
261 def _ellipsis_match(want, got):
263 Essentially the only subtle case:
264 >>> _ellipsis_match('aa...aa', 'aaa')
265 False
267 if ELLIPSIS_MARKER not in want:
268 return want == got
270 # Find "the real" strings.
271 ws = want.split(ELLIPSIS_MARKER)
272 assert len(ws) >= 2
274 # Deal with exact matches possibly needed at one or both ends.
275 startpos, endpos = 0, len(got)
276 w = ws[0]
277 if w: # starts with exact match
278 if got.startswith(w):
279 startpos = len(w)
280 del ws[0]
281 else:
282 return False
283 w = ws[-1]
284 if w: # ends with exact match
285 if got.endswith(w):
286 endpos -= len(w)
287 del ws[-1]
288 else:
289 return False
291 if startpos > endpos:
292 # Exact end matches required more characters than we have, as in
293 # _ellipsis_match('aa...aa', 'aaa')
294 return False
296 # For the rest, we only need to find the leftmost non-overlapping
297 # match for each piece. If there's no overall match that way alone,
298 # there's no overall match period.
299 for w in ws:
300 # w may be '' at times, if there are consecutive ellipses, or
301 # due to an ellipsis at the start or end of `want`. That's OK.
302 # Search for an empty string succeeds, and doesn't change startpos.
303 startpos = got.find(w, startpos, endpos)
304 if startpos < 0:
305 return False
306 startpos += len(w)
308 return True
310 def _comment_line(line):
311 "Return a commented form of the given line"
312 line = line.rstrip()
313 if line:
314 return '# '+line
315 else:
316 return '#'
318 class _OutputRedirectingPdb(pdb.Pdb):
320 A specialized version of the python debugger that redirects stdout
321 to a given stream when interacting with the user. Stdout is *not*
322 redirected when traced code is executed.
324 def __init__(self, out):
325 self.__out = out
326 self.__debugger_used = False
327 pdb.Pdb.__init__(self, stdout=out)
329 def set_trace(self, frame=None):
330 self.__debugger_used = True
331 if frame is None:
332 frame = sys._getframe().f_back
333 pdb.Pdb.set_trace(self, frame)
335 def set_continue(self):
336 # Calling set_continue unconditionally would break unit test
337 # coverage reporting, as Bdb.set_continue calls sys.settrace(None).
338 if self.__debugger_used:
339 pdb.Pdb.set_continue(self)
341 def trace_dispatch(self, *args):
342 # Redirect stdout to the given stream.
343 save_stdout = sys.stdout
344 sys.stdout = self.__out
345 # Call Pdb's trace dispatch method.
346 try:
347 return pdb.Pdb.trace_dispatch(self, *args)
348 finally:
349 sys.stdout = save_stdout
351 # [XX] Normalize with respect to os.path.pardir?
352 def _module_relative_path(module, path):
353 if not inspect.ismodule(module):
354 raise TypeError, 'Expected a module: %r' % module
355 if path.startswith('/'):
356 raise ValueError, 'Module-relative files may not have absolute paths'
358 # Find the base directory for the path.
359 if hasattr(module, '__file__'):
360 # A normal module/package
361 basedir = os.path.split(module.__file__)[0]
362 elif module.__name__ == '__main__':
363 # An interactive session.
364 if len(sys.argv)>0 and sys.argv[0] != '':
365 basedir = os.path.split(sys.argv[0])[0]
366 else:
367 basedir = os.curdir
368 else:
369 # A module w/o __file__ (this includes builtins)
370 raise ValueError("Can't resolve paths relative to the module " +
371 module + " (it has no __file__)")
373 # Combine the base directory and the path.
374 return os.path.join(basedir, *(path.split('/')))
376 ######################################################################
377 ## 2. Example & DocTest
378 ######################################################################
379 ## - An "example" is a <source, want> pair, where "source" is a
380 ## fragment of source code, and "want" is the expected output for
381 ## "source." The Example class also includes information about
382 ## where the example was extracted from.
384 ## - A "doctest" is a collection of examples, typically extracted from
385 ## a string (such as an object's docstring). The DocTest class also
386 ## includes information about where the string was extracted from.
388 class Example:
390 A single doctest example, consisting of source code and expected
391 output. `Example` defines the following attributes:
393 - source: A single Python statement, always ending with a newline.
394 The constructor adds a newline if needed.
396 - want: The expected output from running the source code (either
397 from stdout, or a traceback in case of exception). `want` ends
398 with a newline unless it's empty, in which case it's an empty
399 string. The constructor adds a newline if needed.
401 - exc_msg: The exception message generated by the example, if
402 the example is expected to generate an exception; or `None` if
403 it is not expected to generate an exception. This exception
404 message is compared against the return value of
405 `traceback.format_exception_only()`. `exc_msg` ends with a
406 newline unless it's `None`. The constructor adds a newline
407 if needed.
409 - lineno: The line number within the DocTest string containing
410 this Example where the Example begins. This line number is
411 zero-based, with respect to the beginning of the DocTest.
413 - indent: The example's indentation in the DocTest string.
414 I.e., the number of space characters that preceed the
415 example's first prompt.
417 - options: A dictionary mapping from option flags to True or
418 False, which is used to override default options for this
419 example. Any option flags not contained in this dictionary
420 are left at their default value (as specified by the
421 DocTestRunner's optionflags). By default, no options are set.
423 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
424 options=None):
425 # Normalize inputs.
426 if not source.endswith('\n'):
427 source += '\n'
428 if want and not want.endswith('\n'):
429 want += '\n'
430 if exc_msg is not None and not exc_msg.endswith('\n'):
431 exc_msg += '\n'
432 # Store properties.
433 self.source = source
434 self.want = want
435 self.lineno = lineno
436 self.indent = indent
437 if options is None: options = {}
438 self.options = options
439 self.exc_msg = exc_msg
441 class DocTest:
443 A collection of doctest examples that should be run in a single
444 namespace. Each `DocTest` defines the following attributes:
446 - examples: the list of examples.
448 - globs: The namespace (aka globals) that the examples should
449 be run in.
451 - name: A name identifying the DocTest (typically, the name of
452 the object whose docstring this DocTest was extracted from).
454 - filename: The name of the file that this DocTest was extracted
455 from, or `None` if the filename is unknown.
457 - lineno: The line number within filename where this DocTest
458 begins, or `None` if the line number is unavailable. This
459 line number is zero-based, with respect to the beginning of
460 the file.
462 - docstring: The string that the examples were extracted from,
463 or `None` if the string is unavailable.
465 def __init__(self, examples, globs, name, filename, lineno, docstring):
467 Create a new DocTest containing the given examples. The
468 DocTest's globals are initialized with a copy of `globs`.
470 assert not isinstance(examples, basestring), \
471 "DocTest no longer accepts str; use DocTestParser instead"
472 self.examples = examples
473 self.docstring = docstring
474 self.globs = globs.copy()
475 self.name = name
476 self.filename = filename
477 self.lineno = lineno
479 def __repr__(self):
480 if len(self.examples) == 0:
481 examples = 'no examples'
482 elif len(self.examples) == 1:
483 examples = '1 example'
484 else:
485 examples = '%d examples' % len(self.examples)
486 return ('<DocTest %s from %s:%s (%s)>' %
487 (self.name, self.filename, self.lineno, examples))
490 # This lets us sort tests by name:
491 def __cmp__(self, other):
492 if not isinstance(other, DocTest):
493 return -1
494 return cmp((self.name, self.filename, self.lineno, id(self)),
495 (other.name, other.filename, other.lineno, id(other)))
497 ######################################################################
498 ## 3. DocTestParser
499 ######################################################################
501 class DocTestParser:
503 A class used to parse strings containing doctest examples.
505 # This regular expression is used to find doctest examples in a
506 # string. It defines three groups: `source` is the source code
507 # (including leading indentation and prompts); `indent` is the
508 # indentation of the first (PS1) line of the source code; and
509 # `want` is the expected output (including leading indentation).
510 _EXAMPLE_RE = re.compile(r'''
511 # Source consists of a PS1 line followed by zero or more PS2 lines.
512 (?P<source>
513 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line
514 (?:\n [ ]* \.\.\. .*)*) # PS2 lines
516 # Want consists of any non-blank lines that do not start with PS1.
517 (?P<want> (?:(?![ ]*$) # Not a blank line
518 (?![ ]*>>>) # Not a line starting with PS1
519 .*$\n? # But any other line
521 ''', re.MULTILINE | re.VERBOSE)
523 # A regular expression for handling `want` strings that contain
524 # expected exceptions. It divides `want` into three pieces:
525 # - the traceback header line (`hdr`)
526 # - the traceback stack (`stack`)
527 # - the exception message (`msg`), as generated by
528 # traceback.format_exception_only()
529 # `msg` may have multiple lines. We assume/require that the
530 # exception message is the first non-indented line starting with a word
531 # character following the traceback header line.
532 _EXCEPTION_RE = re.compile(r"""
533 # Grab the traceback header. Different versions of Python have
534 # said different things on the first traceback line.
535 ^(?P<hdr> Traceback\ \(
536 (?: most\ recent\ call\ last
537 | innermost\ last
538 ) \) :
540 \s* $ # toss trailing whitespace on the header.
541 (?P<stack> .*?) # don't blink: absorb stuff until...
542 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum.
543 """, re.VERBOSE | re.MULTILINE | re.DOTALL)
545 # A callable returning a true value iff its argument is a blank line
546 # or contains a single comment.
547 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match
549 def parse(self, string, name='<string>'):
551 Divide the given string into examples and intervening text,
552 and return them as a list of alternating Examples and strings.
553 Line numbers for the Examples are 0-based. The optional
554 argument `name` is a name identifying this string, and is only
555 used for error messages.
557 string = string.expandtabs()
558 # If all lines begin with the same indentation, then strip it.
559 min_indent = self._min_indent(string)
560 if min_indent > 0:
561 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
563 output = []
564 charno, lineno = 0, 0
565 # Find all doctest examples in the string:
566 for m in self._EXAMPLE_RE.finditer(string):
567 # Add the pre-example text to `output`.
568 output.append(string[charno:m.start()])
569 # Update lineno (lines before this example)
570 lineno += string.count('\n', charno, m.start())
571 # Extract info from the regexp match.
572 (source, options, want, exc_msg) = \
573 self._parse_example(m, name, lineno)
574 # Create an Example, and add it to the list.
575 if not self._IS_BLANK_OR_COMMENT(source):
576 output.append( Example(source, want, exc_msg,
577 lineno=lineno,
578 indent=min_indent+len(m.group('indent')),
579 options=options) )
580 # Update lineno (lines inside this example)
581 lineno += string.count('\n', m.start(), m.end())
582 # Update charno.
583 charno = m.end()
584 # Add any remaining post-example text to `output`.
585 output.append(string[charno:])
586 return output
588 def get_doctest(self, string, globs, name, filename, lineno):
590 Extract all doctest examples from the given string, and
591 collect them into a `DocTest` object.
593 `globs`, `name`, `filename`, and `lineno` are attributes for
594 the new `DocTest` object. See the documentation for `DocTest`
595 for more information.
597 return DocTest(self.get_examples(string, name), globs,
598 name, filename, lineno, string)
600 def get_examples(self, string, name='<string>'):
602 Extract all doctest examples from the given string, and return
603 them as a list of `Example` objects. Line numbers are
604 0-based, because it's most common in doctests that nothing
605 interesting appears on the same line as opening triple-quote,
606 and so the first interesting line is called \"line 1\" then.
608 The optional argument `name` is a name identifying this
609 string, and is only used for error messages.
611 return [x for x in self.parse(string, name)
612 if isinstance(x, Example)]
614 def _parse_example(self, m, name, lineno):
616 Given a regular expression match from `_EXAMPLE_RE` (`m`),
617 return a pair `(source, want)`, where `source` is the matched
618 example's source code (with prompts and indentation stripped);
619 and `want` is the example's expected output (with indentation
620 stripped).
622 `name` is the string's name, and `lineno` is the line number
623 where the example starts; both are used for error messages.
625 # Get the example's indentation level.
626 indent = len(m.group('indent'))
628 # Divide source into lines; check that they're properly
629 # indented; and then strip their indentation & prompts.
630 source_lines = m.group('source').split('\n')
631 self._check_prompt_blank(source_lines, indent, name, lineno)
632 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno)
633 source = '\n'.join([sl[indent+4:] for sl in source_lines])
635 # Divide want into lines; check that it's properly indented; and
636 # then strip the indentation. Spaces before the last newline should
637 # be preserved, so plain rstrip() isn't good enough.
638 want = m.group('want')
639 want_lines = want.split('\n')
640 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
641 del want_lines[-1] # forget final newline & spaces after it
642 self._check_prefix(want_lines, ' '*indent, name,
643 lineno + len(source_lines))
644 want = '\n'.join([wl[indent:] for wl in want_lines])
646 # If `want` contains a traceback message, then extract it.
647 m = self._EXCEPTION_RE.match(want)
648 if m:
649 exc_msg = m.group('msg')
650 else:
651 exc_msg = None
653 # Extract options from the source.
654 options = self._find_options(source, name, lineno)
656 return source, options, want, exc_msg
658 # This regular expression looks for option directives in the
659 # source code of an example. Option directives are comments
660 # starting with "doctest:". Warning: this may give false
661 # positives for string-literals that contain the string
662 # "#doctest:". Eliminating these false positives would require
663 # actually parsing the string; but we limit them by ignoring any
664 # line containing "#doctest:" that is *followed* by a quote mark.
665 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$',
666 re.MULTILINE)
668 def _find_options(self, source, name, lineno):
670 Return a dictionary containing option overrides extracted from
671 option directives in the given source string.
673 `name` is the string's name, and `lineno` is the line number
674 where the example starts; both are used for error messages.
676 options = {}
677 # (note: with the current regexp, this will match at most once:)
678 for m in self._OPTION_DIRECTIVE_RE.finditer(source):
679 option_strings = m.group(1).replace(',', ' ').split()
680 for option in option_strings:
681 if (option[0] not in '+-' or
682 option[1:] not in OPTIONFLAGS_BY_NAME):
683 raise ValueError('line %r of the doctest for %s '
684 'has an invalid option: %r' %
685 (lineno+1, name, option))
686 flag = OPTIONFLAGS_BY_NAME[option[1:]]
687 options[flag] = (option[0] == '+')
688 if options and self._IS_BLANK_OR_COMMENT(source):
689 raise ValueError('line %r of the doctest for %s has an option '
690 'directive on a line with no example: %r' %
691 (lineno, name, source))
692 return options
694 # This regular expression finds the indentation of every non-blank
695 # line in a string.
696 _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE)
698 def _min_indent(self, s):
699 "Return the minimum indentation of any non-blank line in `s`"
700 indents = [len(indent) for indent in self._INDENT_RE.findall(s)]
701 if len(indents) > 0:
702 return min(indents)
703 else:
704 return 0
706 def _check_prompt_blank(self, lines, indent, name, lineno):
708 Given the lines of a source string (including prompts and
709 leading indentation), check to make sure that every prompt is
710 followed by a space character. If any line is not followed by
711 a space character, then raise ValueError.
713 for i, line in enumerate(lines):
714 if len(line) >= indent+4 and line[indent+3] != ' ':
715 raise ValueError('line %r of the docstring for %s '
716 'lacks blank after %s: %r' %
717 (lineno+i+1, name,
718 line[indent:indent+3], line))
720 def _check_prefix(self, lines, prefix, name, lineno):
722 Check that every line in the given list starts with the given
723 prefix; if any line does not, then raise a ValueError.
725 for i, line in enumerate(lines):
726 if line and not line.startswith(prefix):
727 raise ValueError('line %r of the docstring for %s has '
728 'inconsistent leading whitespace: %r' %
729 (lineno+i+1, name, line))
732 ######################################################################
733 ## 4. DocTest Finder
734 ######################################################################
736 class DocTestFinder:
738 A class used to extract the DocTests that are relevant to a given
739 object, from its docstring and the docstrings of its contained
740 objects. Doctests can currently be extracted from the following
741 object types: modules, functions, classes, methods, staticmethods,
742 classmethods, and properties.
745 def __init__(self, verbose=False, parser=DocTestParser(),
746 recurse=True, exclude_empty=True):
748 Create a new doctest finder.
750 The optional argument `parser` specifies a class or
751 function that should be used to create new DocTest objects (or
752 objects that implement the same interface as DocTest). The
753 signature for this factory function should match the signature
754 of the DocTest constructor.
756 If the optional argument `recurse` is false, then `find` will
757 only examine the given object, and not any contained objects.
759 If the optional argument `exclude_empty` is false, then `find`
760 will include tests for objects with empty docstrings.
762 self._parser = parser
763 self._verbose = verbose
764 self._recurse = recurse
765 self._exclude_empty = exclude_empty
767 def find(self, obj, name=None, module=None, globs=None, extraglobs=None):
769 Return a list of the DocTests that are defined by the given
770 object's docstring, or by any of its contained objects'
771 docstrings.
773 The optional parameter `module` is the module that contains
774 the given object. If the module is not specified or is None, then
775 the test finder will attempt to automatically determine the
776 correct module. The object's module is used:
778 - As a default namespace, if `globs` is not specified.
779 - To prevent the DocTestFinder from extracting DocTests
780 from objects that are imported from other modules.
781 - To find the name of the file containing the object.
782 - To help find the line number of the object within its
783 file.
785 Contained objects whose module does not match `module` are ignored.
787 If `module` is False, no attempt to find the module will be made.
788 This is obscure, of use mostly in tests: if `module` is False, or
789 is None but cannot be found automatically, then all objects are
790 considered to belong to the (non-existent) module, so all contained
791 objects will (recursively) be searched for doctests.
793 The globals for each DocTest is formed by combining `globs`
794 and `extraglobs` (bindings in `extraglobs` override bindings
795 in `globs`). A new copy of the globals dictionary is created
796 for each DocTest. If `globs` is not specified, then it
797 defaults to the module's `__dict__`, if specified, or {}
798 otherwise. If `extraglobs` is not specified, then it defaults
799 to {}.
802 # If name was not specified, then extract it from the object.
803 if name is None:
804 name = getattr(obj, '__name__', None)
805 if name is None:
806 raise ValueError("DocTestFinder.find: name must be given "
807 "when obj.__name__ doesn't exist: %r" %
808 (type(obj),))
810 # Find the module that contains the given object (if obj is
811 # a module, then module=obj.). Note: this may fail, in which
812 # case module will be None.
813 if module is False:
814 module = None
815 elif module is None:
816 module = inspect.getmodule(obj)
818 # Read the module's source code. This is used by
819 # DocTestFinder._find_lineno to find the line number for a
820 # given object's docstring.
821 try:
822 file = inspect.getsourcefile(obj) or inspect.getfile(obj)
823 if module is not None:
824 # Supply the module globals in case the module was
825 # originally loaded via a PEP 302 loader and
826 # file is not a valid filesystem path
827 source_lines = linecache.getlines(file, module.__dict__)
828 else:
829 # No access to a loader, so assume it's a normal
830 # filesystem path
831 source_lines = linecache.getlines(file)
832 if not source_lines:
833 source_lines = None
834 except TypeError:
835 source_lines = None
837 # Initialize globals, and merge in extraglobs.
838 if globs is None:
839 if module is None:
840 globs = {}
841 else:
842 globs = module.__dict__.copy()
843 else:
844 globs = globs.copy()
845 if extraglobs is not None:
846 globs.update(extraglobs)
847 if '__name__' not in globs:
848 globs['__name__'] = '__main__' # provide a default module name
850 # Recursively expore `obj`, extracting DocTests.
851 tests = []
852 self._find(tests, obj, name, module, source_lines, globs, {})
853 # Sort the tests by alpha order of names, for consistency in
854 # verbose-mode output. This was a feature of doctest in Pythons
855 # <= 2.3 that got lost by accident in 2.4. It was repaired in
856 # 2.4.4 and 2.5.
857 tests.sort()
858 return tests
860 def _from_module(self, module, object):
862 Return true if the given object is defined in the given
863 module.
865 if module is None:
866 return True
867 elif inspect.getmodule(object) is not None:
868 return module is inspect.getmodule(object)
869 elif inspect.isfunction(object):
870 return module.__dict__ is object.func_globals
871 elif inspect.isclass(object):
872 return module.__name__ == object.__module__
873 elif hasattr(object, '__module__'):
874 return module.__name__ == object.__module__
875 elif isinstance(object, property):
876 return True # [XX] no way not be sure.
877 else:
878 raise ValueError("object must be a class or function")
880 def _find(self, tests, obj, name, module, source_lines, globs, seen):
882 Find tests for the given object and any contained objects, and
883 add them to `tests`.
885 if self._verbose:
886 print 'Finding tests in %s' % name
888 # If we've already processed this object, then ignore it.
889 if id(obj) in seen:
890 return
891 seen[id(obj)] = 1
893 # Find a test for this object, and add it to the list of tests.
894 test = self._get_test(obj, name, module, globs, source_lines)
895 if test is not None:
896 tests.append(test)
898 # Look for tests in a module's contained objects.
899 if inspect.ismodule(obj) and self._recurse:
900 for valname, val in obj.__dict__.items():
901 valname = '%s.%s' % (name, valname)
902 # Recurse to functions & classes.
903 if ((inspect.isfunction(val) or inspect.isclass(val)) and
904 self._from_module(module, val)):
905 self._find(tests, val, valname, module, source_lines,
906 globs, seen)
908 # Look for tests in a module's __test__ dictionary.
909 if inspect.ismodule(obj) and self._recurse:
910 for valname, val in getattr(obj, '__test__', {}).items():
911 if not isinstance(valname, basestring):
912 raise ValueError("DocTestFinder.find: __test__ keys "
913 "must be strings: %r" %
914 (type(valname),))
915 if not (inspect.isfunction(val) or inspect.isclass(val) or
916 inspect.ismethod(val) or inspect.ismodule(val) or
917 isinstance(val, basestring)):
918 raise ValueError("DocTestFinder.find: __test__ values "
919 "must be strings, functions, methods, "
920 "classes, or modules: %r" %
921 (type(val),))
922 valname = '%s.__test__.%s' % (name, valname)
923 self._find(tests, val, valname, module, source_lines,
924 globs, seen)
926 # Look for tests in a class's contained objects.
927 if inspect.isclass(obj) and self._recurse:
928 for valname, val in obj.__dict__.items():
929 # Special handling for staticmethod/classmethod.
930 if isinstance(val, staticmethod):
931 val = getattr(obj, valname)
932 if isinstance(val, classmethod):
933 val = getattr(obj, valname).im_func
935 # Recurse to methods, properties, and nested classes.
936 if ((inspect.isfunction(val) or inspect.isclass(val) or
937 isinstance(val, property)) and
938 self._from_module(module, val)):
939 valname = '%s.%s' % (name, valname)
940 self._find(tests, val, valname, module, source_lines,
941 globs, seen)
943 def _get_test(self, obj, name, module, globs, source_lines):
945 Return a DocTest for the given object, if it defines a docstring;
946 otherwise, return None.
948 # Extract the object's docstring. If it doesn't have one,
949 # then return None (no test for this object).
950 if isinstance(obj, basestring):
951 docstring = obj
952 else:
953 try:
954 if obj.__doc__ is None:
955 docstring = ''
956 else:
957 docstring = obj.__doc__
958 if not isinstance(docstring, basestring):
959 docstring = str(docstring)
960 except (TypeError, AttributeError):
961 docstring = ''
963 # Find the docstring's location in the file.
964 lineno = self._find_lineno(obj, source_lines)
966 # Don't bother if the docstring is empty.
967 if self._exclude_empty and not docstring:
968 return None
970 # Return a DocTest for this object.
971 if module is None:
972 filename = None
973 else:
974 filename = getattr(module, '__file__', module.__name__)
975 if filename[-4:] in (".pyc", ".pyo"):
976 filename = filename[:-1]
977 return self._parser.get_doctest(docstring, globs, name,
978 filename, lineno)
980 def _find_lineno(self, obj, source_lines):
982 Return a line number of the given object's docstring. Note:
983 this method assumes that the object has a docstring.
985 lineno = None
987 # Find the line number for modules.
988 if inspect.ismodule(obj):
989 lineno = 0
991 # Find the line number for classes.
992 # Note: this could be fooled if a class is defined multiple
993 # times in a single file.
994 if inspect.isclass(obj):
995 if source_lines is None:
996 return None
997 pat = re.compile(r'^\s*class\s*%s\b' %
998 getattr(obj, '__name__', '-'))
999 for i, line in enumerate(source_lines):
1000 if pat.match(line):
1001 lineno = i
1002 break
1004 # Find the line number for functions & methods.
1005 if inspect.ismethod(obj): obj = obj.im_func
1006 if inspect.isfunction(obj): obj = obj.func_code
1007 if inspect.istraceback(obj): obj = obj.tb_frame
1008 if inspect.isframe(obj): obj = obj.f_code
1009 if inspect.iscode(obj):
1010 lineno = getattr(obj, 'co_firstlineno', None)-1
1012 # Find the line number where the docstring starts. Assume
1013 # that it's the first line that begins with a quote mark.
1014 # Note: this could be fooled by a multiline function
1015 # signature, where a continuation line begins with a quote
1016 # mark.
1017 if lineno is not None:
1018 if source_lines is None:
1019 return lineno+1
1020 pat = re.compile('(^|.*:)\s*\w*("|\')')
1021 for lineno in range(lineno, len(source_lines)):
1022 if pat.match(source_lines[lineno]):
1023 return lineno
1025 # We couldn't find the line number.
1026 return None
1028 ######################################################################
1029 ## 5. DocTest Runner
1030 ######################################################################
1032 class DocTestRunner:
1034 A class used to run DocTest test cases, and accumulate statistics.
1035 The `run` method is used to process a single DocTest case. It
1036 returns a tuple `(f, t)`, where `t` is the number of test cases
1037 tried, and `f` is the number of test cases that failed.
1039 >>> tests = DocTestFinder().find(_TestClass)
1040 >>> runner = DocTestRunner(verbose=False)
1041 >>> tests.sort(key = lambda test: test.name)
1042 >>> for test in tests:
1043 ... print test.name, '->', runner.run(test)
1044 _TestClass -> TestResults(failed=0, attempted=2)
1045 _TestClass.__init__ -> TestResults(failed=0, attempted=2)
1046 _TestClass.get -> TestResults(failed=0, attempted=2)
1047 _TestClass.square -> TestResults(failed=0, attempted=1)
1049 The `summarize` method prints a summary of all the test cases that
1050 have been run by the runner, and returns an aggregated `(f, t)`
1051 tuple:
1053 >>> runner.summarize(verbose=1)
1054 4 items passed all tests:
1055 2 tests in _TestClass
1056 2 tests in _TestClass.__init__
1057 2 tests in _TestClass.get
1058 1 tests in _TestClass.square
1059 7 tests in 4 items.
1060 7 passed and 0 failed.
1061 Test passed.
1062 TestResults(failed=0, attempted=7)
1064 The aggregated number of tried examples and failed examples is
1065 also available via the `tries` and `failures` attributes:
1067 >>> runner.tries
1069 >>> runner.failures
1072 The comparison between expected outputs and actual outputs is done
1073 by an `OutputChecker`. This comparison may be customized with a
1074 number of option flags; see the documentation for `testmod` for
1075 more information. If the option flags are insufficient, then the
1076 comparison may also be customized by passing a subclass of
1077 `OutputChecker` to the constructor.
1079 The test runner's display output can be controlled in two ways.
1080 First, an output function (`out) can be passed to
1081 `TestRunner.run`; this function will be called with strings that
1082 should be displayed. It defaults to `sys.stdout.write`. If
1083 capturing the output is not sufficient, then the display output
1084 can be also customized by subclassing DocTestRunner, and
1085 overriding the methods `report_start`, `report_success`,
1086 `report_unexpected_exception`, and `report_failure`.
1088 # This divider string is used to separate failure messages, and to
1089 # separate sections of the summary.
1090 DIVIDER = "*" * 70
1092 def __init__(self, checker=None, verbose=None, optionflags=0):
1094 Create a new test runner.
1096 Optional keyword arg `checker` is the `OutputChecker` that
1097 should be used to compare the expected outputs and actual
1098 outputs of doctest examples.
1100 Optional keyword arg 'verbose' prints lots of stuff if true,
1101 only failures if false; by default, it's true iff '-v' is in
1102 sys.argv.
1104 Optional argument `optionflags` can be used to control how the
1105 test runner compares expected output to actual output, and how
1106 it displays failures. See the documentation for `testmod` for
1107 more information.
1109 self._checker = checker or OutputChecker()
1110 if verbose is None:
1111 verbose = '-v' in sys.argv
1112 self._verbose = verbose
1113 self.optionflags = optionflags
1114 self.original_optionflags = optionflags
1116 # Keep track of the examples we've run.
1117 self.tries = 0
1118 self.failures = 0
1119 self._name2ft = {}
1121 # Create a fake output target for capturing doctest output.
1122 self._fakeout = _SpoofOut()
1124 #/////////////////////////////////////////////////////////////////
1125 # Reporting methods
1126 #/////////////////////////////////////////////////////////////////
1128 def report_start(self, out, test, example):
1130 Report that the test runner is about to process the given
1131 example. (Only displays a message if verbose=True)
1133 if self._verbose:
1134 if example.want:
1135 out('Trying:\n' + _indent(example.source) +
1136 'Expecting:\n' + _indent(example.want))
1137 else:
1138 out('Trying:\n' + _indent(example.source) +
1139 'Expecting nothing\n')
1141 def report_success(self, out, test, example, got):
1143 Report that the given example ran successfully. (Only
1144 displays a message if verbose=True)
1146 if self._verbose:
1147 out("ok\n")
1149 def report_failure(self, out, test, example, got):
1151 Report that the given example failed.
1153 out(self._failure_header(test, example) +
1154 self._checker.output_difference(example, got, self.optionflags))
1156 def report_unexpected_exception(self, out, test, example, exc_info):
1158 Report that the given example raised an unexpected exception.
1160 out(self._failure_header(test, example) +
1161 'Exception raised:\n' + _indent(_exception_traceback(exc_info)))
1163 def _failure_header(self, test, example):
1164 out = [self.DIVIDER]
1165 if test.filename:
1166 if test.lineno is not None and example.lineno is not None:
1167 lineno = test.lineno + example.lineno + 1
1168 else:
1169 lineno = '?'
1170 out.append('File "%s", line %s, in %s' %
1171 (test.filename, lineno, test.name))
1172 else:
1173 out.append('Line %s, in %s' % (example.lineno+1, test.name))
1174 out.append('Failed example:')
1175 source = example.source
1176 out.append(_indent(source))
1177 return '\n'.join(out)
1179 #/////////////////////////////////////////////////////////////////
1180 # DocTest Running
1181 #/////////////////////////////////////////////////////////////////
1183 def __run(self, test, compileflags, out):
1185 Run the examples in `test`. Write the outcome of each example
1186 with one of the `DocTestRunner.report_*` methods, using the
1187 writer function `out`. `compileflags` is the set of compiler
1188 flags that should be used to execute examples. Return a tuple
1189 `(f, t)`, where `t` is the number of examples tried, and `f`
1190 is the number of examples that failed. The examples are run
1191 in the namespace `test.globs`.
1193 # Keep track of the number of failures and tries.
1194 failures = tries = 0
1196 # Save the option flags (since option directives can be used
1197 # to modify them).
1198 original_optionflags = self.optionflags
1200 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state
1202 check = self._checker.check_output
1204 # Process each example.
1205 for examplenum, example in enumerate(test.examples):
1207 # If REPORT_ONLY_FIRST_FAILURE is set, then supress
1208 # reporting after the first failure.
1209 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
1210 failures > 0)
1212 # Merge in the example's options.
1213 self.optionflags = original_optionflags
1214 if example.options:
1215 for (optionflag, val) in example.options.items():
1216 if val:
1217 self.optionflags |= optionflag
1218 else:
1219 self.optionflags &= ~optionflag
1221 # If 'SKIP' is set, then skip this example.
1222 if self.optionflags & SKIP:
1223 continue
1225 # Record that we started this example.
1226 tries += 1
1227 if not quiet:
1228 self.report_start(out, test, example)
1230 # Use a special filename for compile(), so we can retrieve
1231 # the source code during interactive debugging (see
1232 # __patched_linecache_getlines).
1233 filename = '<doctest %s[%d]>' % (test.name, examplenum)
1235 # Run the example in the given context (globs), and record
1236 # any exception that gets raised. (But don't intercept
1237 # keyboard interrupts.)
1238 try:
1239 # Don't blink! This is where the user's code gets run.
1240 exec compile(example.source, filename, "single",
1241 compileflags, 1) in test.globs
1242 self.debugger.set_continue() # ==== Example Finished ====
1243 exception = None
1244 except KeyboardInterrupt:
1245 raise
1246 except:
1247 exception = sys.exc_info()
1248 self.debugger.set_continue() # ==== Example Finished ====
1250 got = self._fakeout.getvalue() # the actual output
1251 self._fakeout.truncate(0)
1252 outcome = FAILURE # guilty until proved innocent or insane
1254 # If the example executed without raising any exceptions,
1255 # verify its output.
1256 if exception is None:
1257 if check(example.want, got, self.optionflags):
1258 outcome = SUCCESS
1260 # The example raised an exception: check if it was expected.
1261 else:
1262 exc_info = sys.exc_info()
1263 exc_msg = traceback.format_exception_only(*exc_info[:2])[-1]
1264 if not quiet:
1265 got += _exception_traceback(exc_info)
1267 # If `example.exc_msg` is None, then we weren't expecting
1268 # an exception.
1269 if example.exc_msg is None:
1270 outcome = BOOM
1272 # We expected an exception: see whether it matches.
1273 elif check(example.exc_msg, exc_msg, self.optionflags):
1274 outcome = SUCCESS
1276 # Another chance if they didn't care about the detail.
1277 elif self.optionflags & IGNORE_EXCEPTION_DETAIL:
1278 m1 = re.match(r'[^:]*:', example.exc_msg)
1279 m2 = re.match(r'[^:]*:', exc_msg)
1280 if m1 and m2 and check(m1.group(0), m2.group(0),
1281 self.optionflags):
1282 outcome = SUCCESS
1284 # Report the outcome.
1285 if outcome is SUCCESS:
1286 if not quiet:
1287 self.report_success(out, test, example, got)
1288 elif outcome is FAILURE:
1289 if not quiet:
1290 self.report_failure(out, test, example, got)
1291 failures += 1
1292 elif outcome is BOOM:
1293 if not quiet:
1294 self.report_unexpected_exception(out, test, example,
1295 exc_info)
1296 failures += 1
1297 else:
1298 assert False, ("unknown outcome", outcome)
1300 # Restore the option flags (in case they were modified)
1301 self.optionflags = original_optionflags
1303 # Record and return the number of failures and tries.
1304 self.__record_outcome(test, failures, tries)
1305 return TestResults(failures, tries)
1307 def __record_outcome(self, test, f, t):
1309 Record the fact that the given DocTest (`test`) generated `f`
1310 failures out of `t` tried examples.
1312 f2, t2 = self._name2ft.get(test.name, (0,0))
1313 self._name2ft[test.name] = (f+f2, t+t2)
1314 self.failures += f
1315 self.tries += t
1317 __LINECACHE_FILENAME_RE = re.compile(r'<doctest '
1318 r'(?P<name>[\w\.]+)'
1319 r'\[(?P<examplenum>\d+)\]>$')
1320 def __patched_linecache_getlines(self, filename, module_globals=None):
1321 m = self.__LINECACHE_FILENAME_RE.match(filename)
1322 if m and m.group('name') == self.test.name:
1323 example = self.test.examples[int(m.group('examplenum'))]
1324 return example.source.splitlines(True)
1325 else:
1326 return self.save_linecache_getlines(filename, module_globals)
1328 def run(self, test, compileflags=None, out=None, clear_globs=True):
1330 Run the examples in `test`, and display the results using the
1331 writer function `out`.
1333 The examples are run in the namespace `test.globs`. If
1334 `clear_globs` is true (the default), then this namespace will
1335 be cleared after the test runs, to help with garbage
1336 collection. If you would like to examine the namespace after
1337 the test completes, then use `clear_globs=False`.
1339 `compileflags` gives the set of flags that should be used by
1340 the Python compiler when running the examples. If not
1341 specified, then it will default to the set of future-import
1342 flags that apply to `globs`.
1344 The output of each example is checked using
1345 `DocTestRunner.check_output`, and the results are formatted by
1346 the `DocTestRunner.report_*` methods.
1348 self.test = test
1350 if compileflags is None:
1351 compileflags = _extract_future_flags(test.globs)
1353 save_stdout = sys.stdout
1354 if out is None:
1355 out = save_stdout.write
1356 sys.stdout = self._fakeout
1358 # Patch pdb.set_trace to restore sys.stdout during interactive
1359 # debugging (so it's not still redirected to self._fakeout).
1360 # Note that the interactive output will go to *our*
1361 # save_stdout, even if that's not the real sys.stdout; this
1362 # allows us to write test cases for the set_trace behavior.
1363 save_set_trace = pdb.set_trace
1364 self.debugger = _OutputRedirectingPdb(save_stdout)
1365 self.debugger.reset()
1366 pdb.set_trace = self.debugger.set_trace
1368 # Patch linecache.getlines, so we can see the example's source
1369 # when we're inside the debugger.
1370 self.save_linecache_getlines = linecache.getlines
1371 linecache.getlines = self.__patched_linecache_getlines
1373 try:
1374 return self.__run(test, compileflags, out)
1375 finally:
1376 sys.stdout = save_stdout
1377 pdb.set_trace = save_set_trace
1378 linecache.getlines = self.save_linecache_getlines
1379 if clear_globs:
1380 test.globs.clear()
1382 #/////////////////////////////////////////////////////////////////
1383 # Summarization
1384 #/////////////////////////////////////////////////////////////////
1385 def summarize(self, verbose=None):
1387 Print a summary of all the test cases that have been run by
1388 this DocTestRunner, and return a tuple `(f, t)`, where `f` is
1389 the total number of failed examples, and `t` is the total
1390 number of tried examples.
1392 The optional `verbose` argument controls how detailed the
1393 summary is. If the verbosity is not specified, then the
1394 DocTestRunner's verbosity is used.
1396 if verbose is None:
1397 verbose = self._verbose
1398 notests = []
1399 passed = []
1400 failed = []
1401 totalt = totalf = 0
1402 for x in self._name2ft.items():
1403 name, (f, t) = x
1404 assert f <= t
1405 totalt += t
1406 totalf += f
1407 if t == 0:
1408 notests.append(name)
1409 elif f == 0:
1410 passed.append( (name, t) )
1411 else:
1412 failed.append(x)
1413 if verbose:
1414 if notests:
1415 print len(notests), "items had no tests:"
1416 notests.sort()
1417 for thing in notests:
1418 print " ", thing
1419 if passed:
1420 print len(passed), "items passed all tests:"
1421 passed.sort()
1422 for thing, count in passed:
1423 print " %3d tests in %s" % (count, thing)
1424 if failed:
1425 print self.DIVIDER
1426 print len(failed), "items had failures:"
1427 failed.sort()
1428 for thing, (f, t) in failed:
1429 print " %3d of %3d in %s" % (f, t, thing)
1430 if verbose:
1431 print totalt, "tests in", len(self._name2ft), "items."
1432 print totalt - totalf, "passed and", totalf, "failed."
1433 if totalf:
1434 print "***Test Failed***", totalf, "failures."
1435 elif verbose:
1436 print "Test passed."
1437 return TestResults(totalf, totalt)
1439 #/////////////////////////////////////////////////////////////////
1440 # Backward compatibility cruft to maintain doctest.master.
1441 #/////////////////////////////////////////////////////////////////
1442 def merge(self, other):
1443 d = self._name2ft
1444 for name, (f, t) in other._name2ft.items():
1445 if name in d:
1446 # Don't print here by default, since doing
1447 # so breaks some of the buildbots
1448 #print "*** DocTestRunner.merge: '" + name + "' in both" \
1449 # " testers; summing outcomes."
1450 f2, t2 = d[name]
1451 f = f + f2
1452 t = t + t2
1453 d[name] = f, t
1455 class OutputChecker:
1457 A class used to check the whether the actual output from a doctest
1458 example matches the expected output. `OutputChecker` defines two
1459 methods: `check_output`, which compares a given pair of outputs,
1460 and returns true if they match; and `output_difference`, which
1461 returns a string describing the differences between two outputs.
1463 def check_output(self, want, got, optionflags):
1465 Return True iff the actual output from an example (`got`)
1466 matches the expected output (`want`). These strings are
1467 always considered to match if they are identical; but
1468 depending on what option flags the test runner is using,
1469 several non-exact match types are also possible. See the
1470 documentation for `TestRunner` for more information about
1471 option flags.
1473 # Handle the common case first, for efficiency:
1474 # if they're string-identical, always return true.
1475 if got == want:
1476 return True
1478 # The values True and False replaced 1 and 0 as the return
1479 # value for boolean comparisons in Python 2.3.
1480 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1):
1481 if (got,want) == ("True\n", "1\n"):
1482 return True
1483 if (got,want) == ("False\n", "0\n"):
1484 return True
1486 # <BLANKLINE> can be used as a special sequence to signify a
1487 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used.
1488 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1489 # Replace <BLANKLINE> in want with a blank line.
1490 want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER),
1491 '', want)
1492 # If a line in got contains only spaces, then remove the
1493 # spaces.
1494 got = re.sub('(?m)^\s*?$', '', got)
1495 if got == want:
1496 return True
1498 # This flag causes doctest to ignore any differences in the
1499 # contents of whitespace strings. Note that this can be used
1500 # in conjunction with the ELLIPSIS flag.
1501 if optionflags & NORMALIZE_WHITESPACE:
1502 got = ' '.join(got.split())
1503 want = ' '.join(want.split())
1504 if got == want:
1505 return True
1507 # The ELLIPSIS flag says to let the sequence "..." in `want`
1508 # match any substring in `got`.
1509 if optionflags & ELLIPSIS:
1510 if _ellipsis_match(want, got):
1511 return True
1513 # We didn't find any match; return false.
1514 return False
1516 # Should we do a fancy diff?
1517 def _do_a_fancy_diff(self, want, got, optionflags):
1518 # Not unless they asked for a fancy diff.
1519 if not optionflags & (REPORT_UDIFF |
1520 REPORT_CDIFF |
1521 REPORT_NDIFF):
1522 return False
1524 # If expected output uses ellipsis, a meaningful fancy diff is
1525 # too hard ... or maybe not. In two real-life failures Tim saw,
1526 # a diff was a major help anyway, so this is commented out.
1527 # [todo] _ellipsis_match() knows which pieces do and don't match,
1528 # and could be the basis for a kick-ass diff in this case.
1529 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want:
1530 ## return False
1532 # ndiff does intraline difference marking, so can be useful even
1533 # for 1-line differences.
1534 if optionflags & REPORT_NDIFF:
1535 return True
1537 # The other diff types need at least a few lines to be helpful.
1538 return want.count('\n') > 2 and got.count('\n') > 2
1540 def output_difference(self, example, got, optionflags):
1542 Return a string describing the differences between the
1543 expected output for a given example (`example`) and the actual
1544 output (`got`). `optionflags` is the set of option flags used
1545 to compare `want` and `got`.
1547 want = example.want
1548 # If <BLANKLINE>s are being used, then replace blank lines
1549 # with <BLANKLINE> in the actual output string.
1550 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1551 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
1553 # Check if we should use diff.
1554 if self._do_a_fancy_diff(want, got, optionflags):
1555 # Split want & got into lines.
1556 want_lines = want.splitlines(True) # True == keep line ends
1557 got_lines = got.splitlines(True)
1558 # Use difflib to find their differences.
1559 if optionflags & REPORT_UDIFF:
1560 diff = difflib.unified_diff(want_lines, got_lines, n=2)
1561 diff = list(diff)[2:] # strip the diff header
1562 kind = 'unified diff with -expected +actual'
1563 elif optionflags & REPORT_CDIFF:
1564 diff = difflib.context_diff(want_lines, got_lines, n=2)
1565 diff = list(diff)[2:] # strip the diff header
1566 kind = 'context diff with expected followed by actual'
1567 elif optionflags & REPORT_NDIFF:
1568 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
1569 diff = list(engine.compare(want_lines, got_lines))
1570 kind = 'ndiff with -expected +actual'
1571 else:
1572 assert 0, 'Bad diff option'
1573 # Remove trailing whitespace on diff output.
1574 diff = [line.rstrip() + '\n' for line in diff]
1575 return 'Differences (%s):\n' % kind + _indent(''.join(diff))
1577 # If we're not using diff, then simply list the expected
1578 # output followed by the actual output.
1579 if want and got:
1580 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got))
1581 elif want:
1582 return 'Expected:\n%sGot nothing\n' % _indent(want)
1583 elif got:
1584 return 'Expected nothing\nGot:\n%s' % _indent(got)
1585 else:
1586 return 'Expected nothing\nGot nothing\n'
1588 class DocTestFailure(Exception):
1589 """A DocTest example has failed in debugging mode.
1591 The exception instance has variables:
1593 - test: the DocTest object being run
1595 - example: the Example object that failed
1597 - got: the actual output
1599 def __init__(self, test, example, got):
1600 self.test = test
1601 self.example = example
1602 self.got = got
1604 def __str__(self):
1605 return str(self.test)
1607 class UnexpectedException(Exception):
1608 """A DocTest example has encountered an unexpected exception
1610 The exception instance has variables:
1612 - test: the DocTest object being run
1614 - example: the Example object that failed
1616 - exc_info: the exception info
1618 def __init__(self, test, example, exc_info):
1619 self.test = test
1620 self.example = example
1621 self.exc_info = exc_info
1623 def __str__(self):
1624 return str(self.test)
1626 class DebugRunner(DocTestRunner):
1627 r"""Run doc tests but raise an exception as soon as there is a failure.
1629 If an unexpected exception occurs, an UnexpectedException is raised.
1630 It contains the test, the example, and the original exception:
1632 >>> runner = DebugRunner(verbose=False)
1633 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
1634 ... {}, 'foo', 'foo.py', 0)
1635 >>> try:
1636 ... runner.run(test)
1637 ... except UnexpectedException, failure:
1638 ... pass
1640 >>> failure.test is test
1641 True
1643 >>> failure.example.want
1644 '42\n'
1646 >>> exc_info = failure.exc_info
1647 >>> raise exc_info[0], exc_info[1], exc_info[2]
1648 Traceback (most recent call last):
1650 KeyError
1652 We wrap the original exception to give the calling application
1653 access to the test and example information.
1655 If the output doesn't match, then a DocTestFailure is raised:
1657 >>> test = DocTestParser().get_doctest('''
1658 ... >>> x = 1
1659 ... >>> x
1660 ... 2
1661 ... ''', {}, 'foo', 'foo.py', 0)
1663 >>> try:
1664 ... runner.run(test)
1665 ... except DocTestFailure, failure:
1666 ... pass
1668 DocTestFailure objects provide access to the test:
1670 >>> failure.test is test
1671 True
1673 As well as to the example:
1675 >>> failure.example.want
1676 '2\n'
1678 and the actual output:
1680 >>> failure.got
1681 '1\n'
1683 If a failure or error occurs, the globals are left intact:
1685 >>> del test.globs['__builtins__']
1686 >>> test.globs
1687 {'x': 1}
1689 >>> test = DocTestParser().get_doctest('''
1690 ... >>> x = 2
1691 ... >>> raise KeyError
1692 ... ''', {}, 'foo', 'foo.py', 0)
1694 >>> runner.run(test)
1695 Traceback (most recent call last):
1697 UnexpectedException: <DocTest foo from foo.py:0 (2 examples)>
1699 >>> del test.globs['__builtins__']
1700 >>> test.globs
1701 {'x': 2}
1703 But the globals are cleared if there is no error:
1705 >>> test = DocTestParser().get_doctest('''
1706 ... >>> x = 2
1707 ... ''', {}, 'foo', 'foo.py', 0)
1709 >>> runner.run(test)
1710 TestResults(failed=0, attempted=1)
1712 >>> test.globs
1717 def run(self, test, compileflags=None, out=None, clear_globs=True):
1718 r = DocTestRunner.run(self, test, compileflags, out, False)
1719 if clear_globs:
1720 test.globs.clear()
1721 return r
1723 def report_unexpected_exception(self, out, test, example, exc_info):
1724 raise UnexpectedException(test, example, exc_info)
1726 def report_failure(self, out, test, example, got):
1727 raise DocTestFailure(test, example, got)
1729 ######################################################################
1730 ## 6. Test Functions
1731 ######################################################################
1732 # These should be backwards compatible.
1734 # For backward compatibility, a global instance of a DocTestRunner
1735 # class, updated by testmod.
1736 master = None
1738 def testmod(m=None, name=None, globs=None, verbose=None,
1739 report=True, optionflags=0, extraglobs=None,
1740 raise_on_error=False, exclude_empty=False):
1741 """m=None, name=None, globs=None, verbose=None, report=True,
1742 optionflags=0, extraglobs=None, raise_on_error=False,
1743 exclude_empty=False
1745 Test examples in docstrings in functions and classes reachable
1746 from module m (or the current module if m is not supplied), starting
1747 with m.__doc__.
1749 Also test examples reachable from dict m.__test__ if it exists and is
1750 not None. m.__test__ maps names to functions, classes and strings;
1751 function and class docstrings are tested even if the name is private;
1752 strings are tested directly, as if they were docstrings.
1754 Return (#failures, #tests).
1756 See doctest.__doc__ for an overview.
1758 Optional keyword arg "name" gives the name of the module; by default
1759 use m.__name__.
1761 Optional keyword arg "globs" gives a dict to be used as the globals
1762 when executing examples; by default, use m.__dict__. A copy of this
1763 dict is actually used for each docstring, so that each docstring's
1764 examples start with a clean slate.
1766 Optional keyword arg "extraglobs" gives a dictionary that should be
1767 merged into the globals that are used to execute examples. By
1768 default, no extra globals are used. This is new in 2.4.
1770 Optional keyword arg "verbose" prints lots of stuff if true, prints
1771 only failures if false; by default, it's true iff "-v" is in sys.argv.
1773 Optional keyword arg "report" prints a summary at the end when true,
1774 else prints nothing at the end. In verbose mode, the summary is
1775 detailed, else very brief (in fact, empty if all tests passed).
1777 Optional keyword arg "optionflags" or's together module constants,
1778 and defaults to 0. This is new in 2.3. Possible values (see the
1779 docs for details):
1781 DONT_ACCEPT_TRUE_FOR_1
1782 DONT_ACCEPT_BLANKLINE
1783 NORMALIZE_WHITESPACE
1784 ELLIPSIS
1785 SKIP
1786 IGNORE_EXCEPTION_DETAIL
1787 REPORT_UDIFF
1788 REPORT_CDIFF
1789 REPORT_NDIFF
1790 REPORT_ONLY_FIRST_FAILURE
1792 Optional keyword arg "raise_on_error" raises an exception on the
1793 first unexpected exception or failure. This allows failures to be
1794 post-mortem debugged.
1796 Advanced tomfoolery: testmod runs methods of a local instance of
1797 class doctest.Tester, then merges the results into (or creates)
1798 global Tester instance doctest.master. Methods of doctest.master
1799 can be called directly too, if you want to do something unusual.
1800 Passing report=0 to testmod is especially useful then, to delay
1801 displaying a summary. Invoke doctest.master.summarize(verbose)
1802 when you're done fiddling.
1804 global master
1806 # If no module was given, then use __main__.
1807 if m is None:
1808 # DWA - m will still be None if this wasn't invoked from the command
1809 # line, in which case the following TypeError is about as good an error
1810 # as we should expect
1811 m = sys.modules.get('__main__')
1813 # Check that we were actually given a module.
1814 if not inspect.ismodule(m):
1815 raise TypeError("testmod: module required; %r" % (m,))
1817 # If no name was given, then use the module's name.
1818 if name is None:
1819 name = m.__name__
1821 # Find, parse, and run all tests in the given module.
1822 finder = DocTestFinder(exclude_empty=exclude_empty)
1824 if raise_on_error:
1825 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
1826 else:
1827 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
1829 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs):
1830 runner.run(test)
1832 if report:
1833 runner.summarize()
1835 if master is None:
1836 master = runner
1837 else:
1838 master.merge(runner)
1840 return TestResults(runner.failures, runner.tries)
1842 def testfile(filename, module_relative=True, name=None, package=None,
1843 globs=None, verbose=None, report=True, optionflags=0,
1844 extraglobs=None, raise_on_error=False, parser=DocTestParser(),
1845 encoding=None):
1847 Test examples in the given file. Return (#failures, #tests).
1849 Optional keyword arg "module_relative" specifies how filenames
1850 should be interpreted:
1852 - If "module_relative" is True (the default), then "filename"
1853 specifies a module-relative path. By default, this path is
1854 relative to the calling module's directory; but if the
1855 "package" argument is specified, then it is relative to that
1856 package. To ensure os-independence, "filename" should use
1857 "/" characters to separate path segments, and should not
1858 be an absolute path (i.e., it may not begin with "/").
1860 - If "module_relative" is False, then "filename" specifies an
1861 os-specific path. The path may be absolute or relative (to
1862 the current working directory).
1864 Optional keyword arg "name" gives the name of the test; by default
1865 use the file's basename.
1867 Optional keyword argument "package" is a Python package or the
1868 name of a Python package whose directory should be used as the
1869 base directory for a module relative filename. If no package is
1870 specified, then the calling module's directory is used as the base
1871 directory for module relative filenames. It is an error to
1872 specify "package" if "module_relative" is False.
1874 Optional keyword arg "globs" gives a dict to be used as the globals
1875 when executing examples; by default, use {}. A copy of this dict
1876 is actually used for each docstring, so that each docstring's
1877 examples start with a clean slate.
1879 Optional keyword arg "extraglobs" gives a dictionary that should be
1880 merged into the globals that are used to execute examples. By
1881 default, no extra globals are used.
1883 Optional keyword arg "verbose" prints lots of stuff if true, prints
1884 only failures if false; by default, it's true iff "-v" is in sys.argv.
1886 Optional keyword arg "report" prints a summary at the end when true,
1887 else prints nothing at the end. In verbose mode, the summary is
1888 detailed, else very brief (in fact, empty if all tests passed).
1890 Optional keyword arg "optionflags" or's together module constants,
1891 and defaults to 0. Possible values (see the docs for details):
1893 DONT_ACCEPT_TRUE_FOR_1
1894 DONT_ACCEPT_BLANKLINE
1895 NORMALIZE_WHITESPACE
1896 ELLIPSIS
1897 SKIP
1898 IGNORE_EXCEPTION_DETAIL
1899 REPORT_UDIFF
1900 REPORT_CDIFF
1901 REPORT_NDIFF
1902 REPORT_ONLY_FIRST_FAILURE
1904 Optional keyword arg "raise_on_error" raises an exception on the
1905 first unexpected exception or failure. This allows failures to be
1906 post-mortem debugged.
1908 Optional keyword arg "parser" specifies a DocTestParser (or
1909 subclass) that should be used to extract tests from the files.
1911 Optional keyword arg "encoding" specifies an encoding that should
1912 be used to convert the file to unicode.
1914 Advanced tomfoolery: testmod runs methods of a local instance of
1915 class doctest.Tester, then merges the results into (or creates)
1916 global Tester instance doctest.master. Methods of doctest.master
1917 can be called directly too, if you want to do something unusual.
1918 Passing report=0 to testmod is especially useful then, to delay
1919 displaying a summary. Invoke doctest.master.summarize(verbose)
1920 when you're done fiddling.
1922 global master
1924 if package and not module_relative:
1925 raise ValueError("Package may only be specified for module-"
1926 "relative paths.")
1928 # Relativize the path
1929 text, filename = _load_testfile(filename, package, module_relative)
1931 # If no name was given, then use the file's name.
1932 if name is None:
1933 name = os.path.basename(filename)
1935 # Assemble the globals.
1936 if globs is None:
1937 globs = {}
1938 else:
1939 globs = globs.copy()
1940 if extraglobs is not None:
1941 globs.update(extraglobs)
1942 if '__name__' not in globs:
1943 globs['__name__'] = '__main__'
1945 if raise_on_error:
1946 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
1947 else:
1948 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
1950 if encoding is not None:
1951 text = text.decode(encoding)
1953 # Read the file, convert it to a test, and run it.
1954 test = parser.get_doctest(text, globs, name, filename, 0)
1955 runner.run(test)
1957 if report:
1958 runner.summarize()
1960 if master is None:
1961 master = runner
1962 else:
1963 master.merge(runner)
1965 return TestResults(runner.failures, runner.tries)
1967 def run_docstring_examples(f, globs, verbose=False, name="NoName",
1968 compileflags=None, optionflags=0):
1970 Test examples in the given object's docstring (`f`), using `globs`
1971 as globals. Optional argument `name` is used in failure messages.
1972 If the optional argument `verbose` is true, then generate output
1973 even if there are no failures.
1975 `compileflags` gives the set of flags that should be used by the
1976 Python compiler when running the examples. If not specified, then
1977 it will default to the set of future-import flags that apply to
1978 `globs`.
1980 Optional keyword arg `optionflags` specifies options for the
1981 testing and output. See the documentation for `testmod` for more
1982 information.
1984 # Find, parse, and run all tests in the given module.
1985 finder = DocTestFinder(verbose=verbose, recurse=False)
1986 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
1987 for test in finder.find(f, name, globs=globs):
1988 runner.run(test, compileflags=compileflags)
1990 ######################################################################
1991 ## 7. Tester
1992 ######################################################################
1993 # This is provided only for backwards compatibility. It's not
1994 # actually used in any way.
1996 class Tester:
1997 def __init__(self, mod=None, globs=None, verbose=None, optionflags=0):
1999 warnings.warn("class Tester is deprecated; "
2000 "use class doctest.DocTestRunner instead",
2001 DeprecationWarning, stacklevel=2)
2002 if mod is None and globs is None:
2003 raise TypeError("Tester.__init__: must specify mod or globs")
2004 if mod is not None and not inspect.ismodule(mod):
2005 raise TypeError("Tester.__init__: mod must be a module; %r" %
2006 (mod,))
2007 if globs is None:
2008 globs = mod.__dict__
2009 self.globs = globs
2011 self.verbose = verbose
2012 self.optionflags = optionflags
2013 self.testfinder = DocTestFinder()
2014 self.testrunner = DocTestRunner(verbose=verbose,
2015 optionflags=optionflags)
2017 def runstring(self, s, name):
2018 test = DocTestParser().get_doctest(s, self.globs, name, None, None)
2019 if self.verbose:
2020 print "Running string", name
2021 (f,t) = self.testrunner.run(test)
2022 if self.verbose:
2023 print f, "of", t, "examples failed in string", name
2024 return TestResults(f,t)
2026 def rundoc(self, object, name=None, module=None):
2027 f = t = 0
2028 tests = self.testfinder.find(object, name, module=module,
2029 globs=self.globs)
2030 for test in tests:
2031 (f2, t2) = self.testrunner.run(test)
2032 (f,t) = (f+f2, t+t2)
2033 return TestResults(f,t)
2035 def rundict(self, d, name, module=None):
2036 import types
2037 m = types.ModuleType(name)
2038 m.__dict__.update(d)
2039 if module is None:
2040 module = False
2041 return self.rundoc(m, name, module)
2043 def run__test__(self, d, name):
2044 import types
2045 m = types.ModuleType(name)
2046 m.__test__ = d
2047 return self.rundoc(m, name)
2049 def summarize(self, verbose=None):
2050 return self.testrunner.summarize(verbose)
2052 def merge(self, other):
2053 self.testrunner.merge(other.testrunner)
2055 ######################################################################
2056 ## 8. Unittest Support
2057 ######################################################################
2059 _unittest_reportflags = 0
2061 def set_unittest_reportflags(flags):
2062 """Sets the unittest option flags.
2064 The old flag is returned so that a runner could restore the old
2065 value if it wished to:
2067 >>> import doctest
2068 >>> old = doctest._unittest_reportflags
2069 >>> doctest.set_unittest_reportflags(REPORT_NDIFF |
2070 ... REPORT_ONLY_FIRST_FAILURE) == old
2071 True
2073 >>> doctest._unittest_reportflags == (REPORT_NDIFF |
2074 ... REPORT_ONLY_FIRST_FAILURE)
2075 True
2077 Only reporting flags can be set:
2079 >>> doctest.set_unittest_reportflags(ELLIPSIS)
2080 Traceback (most recent call last):
2082 ValueError: ('Only reporting flags allowed', 8)
2084 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF |
2085 ... REPORT_ONLY_FIRST_FAILURE)
2086 True
2088 global _unittest_reportflags
2090 if (flags & REPORTING_FLAGS) != flags:
2091 raise ValueError("Only reporting flags allowed", flags)
2092 old = _unittest_reportflags
2093 _unittest_reportflags = flags
2094 return old
2097 class DocTestCase(unittest.TestCase):
2099 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
2100 checker=None):
2102 unittest.TestCase.__init__(self)
2103 self._dt_optionflags = optionflags
2104 self._dt_checker = checker
2105 self._dt_test = test
2106 self._dt_setUp = setUp
2107 self._dt_tearDown = tearDown
2109 def setUp(self):
2110 test = self._dt_test
2112 if self._dt_setUp is not None:
2113 self._dt_setUp(test)
2115 def tearDown(self):
2116 test = self._dt_test
2118 if self._dt_tearDown is not None:
2119 self._dt_tearDown(test)
2121 test.globs.clear()
2123 def runTest(self):
2124 test = self._dt_test
2125 old = sys.stdout
2126 new = StringIO()
2127 optionflags = self._dt_optionflags
2129 if not (optionflags & REPORTING_FLAGS):
2130 # The option flags don't include any reporting flags,
2131 # so add the default reporting flags
2132 optionflags |= _unittest_reportflags
2134 runner = DocTestRunner(optionflags=optionflags,
2135 checker=self._dt_checker, verbose=False)
2137 try:
2138 runner.DIVIDER = "-"*70
2139 failures, tries = runner.run(
2140 test, out=new.write, clear_globs=False)
2141 finally:
2142 sys.stdout = old
2144 if failures:
2145 raise self.failureException(self.format_failure(new.getvalue()))
2147 def format_failure(self, err):
2148 test = self._dt_test
2149 if test.lineno is None:
2150 lineno = 'unknown line number'
2151 else:
2152 lineno = '%s' % test.lineno
2153 lname = '.'.join(test.name.split('.')[-1:])
2154 return ('Failed doctest test for %s\n'
2155 ' File "%s", line %s, in %s\n\n%s'
2156 % (test.name, test.filename, lineno, lname, err)
2159 def debug(self):
2160 r"""Run the test case without results and without catching exceptions
2162 The unit test framework includes a debug method on test cases
2163 and test suites to support post-mortem debugging. The test code
2164 is run in such a way that errors are not caught. This way a
2165 caller can catch the errors and initiate post-mortem debugging.
2167 The DocTestCase provides a debug method that raises
2168 UnexpectedException errors if there is an unexepcted
2169 exception:
2171 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
2172 ... {}, 'foo', 'foo.py', 0)
2173 >>> case = DocTestCase(test)
2174 >>> try:
2175 ... case.debug()
2176 ... except UnexpectedException, failure:
2177 ... pass
2179 The UnexpectedException contains the test, the example, and
2180 the original exception:
2182 >>> failure.test is test
2183 True
2185 >>> failure.example.want
2186 '42\n'
2188 >>> exc_info = failure.exc_info
2189 >>> raise exc_info[0], exc_info[1], exc_info[2]
2190 Traceback (most recent call last):
2192 KeyError
2194 If the output doesn't match, then a DocTestFailure is raised:
2196 >>> test = DocTestParser().get_doctest('''
2197 ... >>> x = 1
2198 ... >>> x
2199 ... 2
2200 ... ''', {}, 'foo', 'foo.py', 0)
2201 >>> case = DocTestCase(test)
2203 >>> try:
2204 ... case.debug()
2205 ... except DocTestFailure, failure:
2206 ... pass
2208 DocTestFailure objects provide access to the test:
2210 >>> failure.test is test
2211 True
2213 As well as to the example:
2215 >>> failure.example.want
2216 '2\n'
2218 and the actual output:
2220 >>> failure.got
2221 '1\n'
2225 self.setUp()
2226 runner = DebugRunner(optionflags=self._dt_optionflags,
2227 checker=self._dt_checker, verbose=False)
2228 runner.run(self._dt_test, clear_globs=False)
2229 self.tearDown()
2231 def id(self):
2232 return self._dt_test.name
2234 def __repr__(self):
2235 name = self._dt_test.name.split('.')
2236 return "%s (%s)" % (name[-1], '.'.join(name[:-1]))
2238 __str__ = __repr__
2240 def shortDescription(self):
2241 return "Doctest: " + self._dt_test.name
2243 def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
2244 **options):
2246 Convert doctest tests for a module to a unittest test suite.
2248 This converts each documentation string in a module that
2249 contains doctest tests to a unittest test case. If any of the
2250 tests in a doc string fail, then the test case fails. An exception
2251 is raised showing the name of the file containing the test and a
2252 (sometimes approximate) line number.
2254 The `module` argument provides the module to be tested. The argument
2255 can be either a module or a module name.
2257 If no argument is given, the calling module is used.
2259 A number of options may be provided as keyword arguments:
2261 setUp
2262 A set-up function. This is called before running the
2263 tests in each file. The setUp function will be passed a DocTest
2264 object. The setUp function can access the test globals as the
2265 globs attribute of the test passed.
2267 tearDown
2268 A tear-down function. This is called after running the
2269 tests in each file. The tearDown function will be passed a DocTest
2270 object. The tearDown function can access the test globals as the
2271 globs attribute of the test passed.
2273 globs
2274 A dictionary containing initial global variables for the tests.
2276 optionflags
2277 A set of doctest option flags expressed as an integer.
2280 if test_finder is None:
2281 test_finder = DocTestFinder()
2283 module = _normalize_module(module)
2284 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs)
2285 if not tests:
2286 # Why do we want to do this? Because it reveals a bug that might
2287 # otherwise be hidden.
2288 raise ValueError(module, "has no tests")
2290 tests.sort()
2291 suite = unittest.TestSuite()
2292 for test in tests:
2293 if len(test.examples) == 0:
2294 continue
2295 if not test.filename:
2296 filename = module.__file__
2297 if filename[-4:] in (".pyc", ".pyo"):
2298 filename = filename[:-1]
2299 test.filename = filename
2300 suite.addTest(DocTestCase(test, **options))
2302 return suite
2304 class DocFileCase(DocTestCase):
2306 def id(self):
2307 return '_'.join(self._dt_test.name.split('.'))
2309 def __repr__(self):
2310 return self._dt_test.filename
2311 __str__ = __repr__
2313 def format_failure(self, err):
2314 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s'
2315 % (self._dt_test.name, self._dt_test.filename, err)
2318 def DocFileTest(path, module_relative=True, package=None,
2319 globs=None, parser=DocTestParser(),
2320 encoding=None, **options):
2321 if globs is None:
2322 globs = {}
2323 else:
2324 globs = globs.copy()
2326 if package and not module_relative:
2327 raise ValueError("Package may only be specified for module-"
2328 "relative paths.")
2330 # Relativize the path.
2331 doc, path = _load_testfile(path, package, module_relative)
2333 if "__file__" not in globs:
2334 globs["__file__"] = path
2336 # Find the file and read it.
2337 name = os.path.basename(path)
2339 # If an encoding is specified, use it to convert the file to unicode
2340 if encoding is not None:
2341 doc = doc.decode(encoding)
2343 # Convert it to a test, and wrap it in a DocFileCase.
2344 test = parser.get_doctest(doc, globs, name, path, 0)
2345 return DocFileCase(test, **options)
2347 def DocFileSuite(*paths, **kw):
2348 """A unittest suite for one or more doctest files.
2350 The path to each doctest file is given as a string; the
2351 interpretation of that string depends on the keyword argument
2352 "module_relative".
2354 A number of options may be provided as keyword arguments:
2356 module_relative
2357 If "module_relative" is True, then the given file paths are
2358 interpreted as os-independent module-relative paths. By
2359 default, these paths are relative to the calling module's
2360 directory; but if the "package" argument is specified, then
2361 they are relative to that package. To ensure os-independence,
2362 "filename" should use "/" characters to separate path
2363 segments, and may not be an absolute path (i.e., it may not
2364 begin with "/").
2366 If "module_relative" is False, then the given file paths are
2367 interpreted as os-specific paths. These paths may be absolute
2368 or relative (to the current working directory).
2370 package
2371 A Python package or the name of a Python package whose directory
2372 should be used as the base directory for module relative paths.
2373 If "package" is not specified, then the calling module's
2374 directory is used as the base directory for module relative
2375 filenames. It is an error to specify "package" if
2376 "module_relative" is False.
2378 setUp
2379 A set-up function. This is called before running the
2380 tests in each file. The setUp function will be passed a DocTest
2381 object. The setUp function can access the test globals as the
2382 globs attribute of the test passed.
2384 tearDown
2385 A tear-down function. This is called after running the
2386 tests in each file. The tearDown function will be passed a DocTest
2387 object. The tearDown function can access the test globals as the
2388 globs attribute of the test passed.
2390 globs
2391 A dictionary containing initial global variables for the tests.
2393 optionflags
2394 A set of doctest option flags expressed as an integer.
2396 parser
2397 A DocTestParser (or subclass) that should be used to extract
2398 tests from the files.
2400 encoding
2401 An encoding that will be used to convert the files to unicode.
2403 suite = unittest.TestSuite()
2405 # We do this here so that _normalize_module is called at the right
2406 # level. If it were called in DocFileTest, then this function
2407 # would be the caller and we might guess the package incorrectly.
2408 if kw.get('module_relative', True):
2409 kw['package'] = _normalize_module(kw.get('package'))
2411 for path in paths:
2412 suite.addTest(DocFileTest(path, **kw))
2414 return suite
2416 ######################################################################
2417 ## 9. Debugging Support
2418 ######################################################################
2420 def script_from_examples(s):
2421 r"""Extract script from text with examples.
2423 Converts text with examples to a Python script. Example input is
2424 converted to regular code. Example output and all other words
2425 are converted to comments:
2427 >>> text = '''
2428 ... Here are examples of simple math.
2430 ... Python has super accurate integer addition
2432 ... >>> 2 + 2
2433 ... 5
2435 ... And very friendly error messages:
2437 ... >>> 1/0
2438 ... To Infinity
2439 ... And
2440 ... Beyond
2442 ... You can use logic if you want:
2444 ... >>> if 0:
2445 ... ... blah
2446 ... ... blah
2447 ... ...
2449 ... Ho hum
2450 ... '''
2452 >>> print script_from_examples(text)
2453 # Here are examples of simple math.
2455 # Python has super accurate integer addition
2457 2 + 2
2458 # Expected:
2459 ## 5
2461 # And very friendly error messages:
2464 # Expected:
2465 ## To Infinity
2466 ## And
2467 ## Beyond
2469 # You can use logic if you want:
2471 if 0:
2472 blah
2473 blah
2475 # Ho hum
2476 <BLANKLINE>
2478 output = []
2479 for piece in DocTestParser().parse(s):
2480 if isinstance(piece, Example):
2481 # Add the example's source code (strip trailing NL)
2482 output.append(piece.source[:-1])
2483 # Add the expected output:
2484 want = piece.want
2485 if want:
2486 output.append('# Expected:')
2487 output += ['## '+l for l in want.split('\n')[:-1]]
2488 else:
2489 # Add non-example text.
2490 output += [_comment_line(l)
2491 for l in piece.split('\n')[:-1]]
2493 # Trim junk on both ends.
2494 while output and output[-1] == '#':
2495 output.pop()
2496 while output and output[0] == '#':
2497 output.pop(0)
2498 # Combine the output, and return it.
2499 # Add a courtesy newline to prevent exec from choking (see bug #1172785)
2500 return '\n'.join(output) + '\n'
2502 def testsource(module, name):
2503 """Extract the test sources from a doctest docstring as a script.
2505 Provide the module (or dotted name of the module) containing the
2506 test to be debugged and the name (within the module) of the object
2507 with the doc string with tests to be debugged.
2509 module = _normalize_module(module)
2510 tests = DocTestFinder().find(module)
2511 test = [t for t in tests if t.name == name]
2512 if not test:
2513 raise ValueError(name, "not found in tests")
2514 test = test[0]
2515 testsrc = script_from_examples(test.docstring)
2516 return testsrc
2518 def debug_src(src, pm=False, globs=None):
2519 """Debug a single doctest docstring, in argument `src`'"""
2520 testsrc = script_from_examples(src)
2521 debug_script(testsrc, pm, globs)
2523 def debug_script(src, pm=False, globs=None):
2524 "Debug a test script. `src` is the script, as a string."
2525 import pdb
2527 # Note that tempfile.NameTemporaryFile() cannot be used. As the
2528 # docs say, a file so created cannot be opened by name a second time
2529 # on modern Windows boxes, and execfile() needs to open it.
2530 srcfilename = tempfile.mktemp(".py", "doctestdebug")
2531 f = open(srcfilename, 'w')
2532 f.write(src)
2533 f.close()
2535 try:
2536 if globs:
2537 globs = globs.copy()
2538 else:
2539 globs = {}
2541 if pm:
2542 try:
2543 execfile(srcfilename, globs, globs)
2544 except:
2545 print sys.exc_info()[1]
2546 pdb.post_mortem(sys.exc_info()[2])
2547 else:
2548 # Note that %r is vital here. '%s' instead can, e.g., cause
2549 # backslashes to get treated as metacharacters on Windows.
2550 pdb.run("execfile(%r)" % srcfilename, globs, globs)
2552 finally:
2553 os.remove(srcfilename)
2555 def debug(module, name, pm=False):
2556 """Debug a single doctest docstring.
2558 Provide the module (or dotted name of the module) containing the
2559 test to be debugged and the name (within the module) of the object
2560 with the docstring with tests to be debugged.
2562 module = _normalize_module(module)
2563 testsrc = testsource(module, name)
2564 debug_script(testsrc, pm, module.__dict__)
2566 ######################################################################
2567 ## 10. Example Usage
2568 ######################################################################
2569 class _TestClass:
2571 A pointless class, for sanity-checking of docstring testing.
2573 Methods:
2574 square()
2575 get()
2577 >>> _TestClass(13).get() + _TestClass(-12).get()
2579 >>> hex(_TestClass(13).square().get())
2580 '0xa9'
2583 def __init__(self, val):
2584 """val -> _TestClass object with associated value val.
2586 >>> t = _TestClass(123)
2587 >>> print t.get()
2591 self.val = val
2593 def square(self):
2594 """square() -> square TestClass's associated value
2596 >>> _TestClass(13).square().get()
2600 self.val = self.val ** 2
2601 return self
2603 def get(self):
2604 """get() -> return TestClass's associated value.
2606 >>> x = _TestClass(-42)
2607 >>> print x.get()
2611 return self.val
2613 __test__ = {"_TestClass": _TestClass,
2614 "string": r"""
2615 Example of a string object, searched as-is.
2616 >>> x = 1; y = 2
2617 >>> x + y, x * y
2618 (3, 2)
2619 """,
2621 "bool-int equivalence": r"""
2622 In 2.2, boolean expressions displayed
2623 0 or 1. By default, we still accept
2624 them. This can be disabled by passing
2625 DONT_ACCEPT_TRUE_FOR_1 to the new
2626 optionflags argument.
2627 >>> 4 == 4
2629 >>> 4 == 4
2630 True
2631 >>> 4 > 4
2633 >>> 4 > 4
2634 False
2635 """,
2637 "blank lines": r"""
2638 Blank lines can be marked with <BLANKLINE>:
2639 >>> print 'foo\n\nbar\n'
2641 <BLANKLINE>
2643 <BLANKLINE>
2644 """,
2646 "ellipsis": r"""
2647 If the ellipsis flag is used, then '...' can be used to
2648 elide substrings in the desired output:
2649 >>> print range(1000) #doctest: +ELLIPSIS
2650 [0, 1, 2, ..., 999]
2651 """,
2653 "whitespace normalization": r"""
2654 If the whitespace normalization flag is used, then
2655 differences in whitespace are ignored.
2656 >>> print range(30) #doctest: +NORMALIZE_WHITESPACE
2657 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
2658 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
2659 27, 28, 29]
2660 """,
2664 def _test():
2665 testfiles = [arg for arg in sys.argv[1:] if arg and arg[0] != '-']
2666 if not testfiles:
2667 name = os.path.basename(sys.argv[0])
2668 if '__loader__' in globals(): # python -m
2669 name, _ = os.path.splitext(name)
2670 print("usage: {0} [-v] file ...".format(name))
2671 return 2
2672 for filename in testfiles:
2673 if filename.endswith(".py"):
2674 # It is a module -- insert its dir into sys.path and try to
2675 # import it. If it is part of a package, that possibly
2676 # won't work because of package imports.
2677 dirname, filename = os.path.split(filename)
2678 sys.path.insert(0, dirname)
2679 m = __import__(filename[:-3])
2680 del sys.path[0]
2681 failures, _ = testmod(m)
2682 else:
2683 failures, _ = testfile(filename, module_relative=False)
2684 if failures:
2685 return 1
2686 return 0
2689 if __name__ == "__main__":
2690 sys.exit(_test())