extras/textwrap.py

   1 """Text wrapping and filling.
   2 """
   3
   4 # Copyright (C) 1999-2001 Gregory P. Ward.
   5 # Copyright (C) 2002, 2003 Python Software Foundation.
   6 # Written by Greg Ward <gward@python.net>
   7
   8 __revision__ = "$Id$"
   9
  10 import string, re
  11 import types
  12
  13 # Do the right thing with boolean values for all known Python versions
  14 # (so this module can be copied to projects that don't depend on Python
  15 # 2.3, e.g. Optik and Docutils).
  16 try:
  17     True, False
  18 except NameError:
  19     (True, False) = (1, 0)
  20
  21 __all__ = ['TextWrapper', 'wrap', 'fill']
  22
  23 # Hardcode the recognized whitespace characters to the US-ASCII
  24 # whitespace characters.  The main reason for doing this is that in
  25 # ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales
  26 # that character winds up in string.whitespace.  Respecting
  27 # string.whitespace in those cases would 1) make textwrap treat 0xa0 the
  28 # same as any other whitespace char, which is clearly wrong (it's a
  29 # *non-breaking* space), 2) possibly cause problems with Unicode,
  30 # since 0xa0 is not in range(128).
  31 _whitespace = '\t\n\x0b\x0c\r '
  32
  33 class TextWrapper:
  34     """
  35     Object for wrapping/filling text.  The public interface consists of
  36     the wrap() and fill() methods; the other methods are just there for
  37     subclasses to override in order to tweak the default behaviour.
  38     If you want to completely replace the main wrapping algorithm,
  39     you'll probably have to override _wrap_chunks().
  40
  41     Several instance attributes control various aspects of wrapping:
  42       width (default: 70)
  43         the maximum width of wrapped lines (unless break_long_words
  44         is false)
  45       initial_indent (default: "")
  46         string that will be prepended to the first line of wrapped
  47         output.  Counts towards the line's width.
  48       subsequent_indent (default: "")
  49         string that will be prepended to all lines save the first
  50         of wrapped output; also counts towards each line's width.
  51       expand_tabs (default: true)
  52         Expand tabs in input text to spaces before further processing.
  53         Each tab will become 1 .. 8 spaces, depending on its position in
  54         its line.  If false, each tab is treated as a single character.
  55       replace_whitespace (default: true)
  56         Replace all whitespace characters in the input text by spaces
  57         after tab expansion.  Note that if expand_tabs is false and
  58         replace_whitespace is true, every tab will be converted to a
  59         single space!
  60       fix_sentence_endings (default: false)
  61         Ensure that sentence-ending punctuation is always followed
  62         by two spaces.  Off by default because the algorithm is
  63         (unavoidably) imperfect.
  64       break_long_words (default: true)
  65         Break words longer than 'width'.  If false, those words will not
  66         be broken, and some lines might be longer than 'width'.
  67     """
  68
  69     whitespace_trans = string.maketrans(_whitespace, ' ' * len(_whitespace))
  70
  71     unicode_whitespace_trans = {}
  72     uspace = ord(u' ')
  73     for x in map(ord, _whitespace):
  74         unicode_whitespace_trans[x] = uspace
  75
  76     # This funky little regex is just the trick for splitting
  77     # text up into word-wrappable chunks.  E.g.
  78     #   "Hello there -- you goof-ball, use the -b option!"
  79     # splits into
  80     #   Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option!
  81     # (after stripping out empty strings).
  82     wordsep_re = re.compile(r'(\s+|'                  # any whitespace
  83                             r'-*\w{2,}-(?=\w{2,})|'   # hyphenated words
  84                             r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))')   # em-dash
  85
  86     # XXX will there be a locale-or-charset-aware version of
  87     # string.lowercase in 2.3?
  88     sentence_end_re = re.compile(r'[%s]'              # lowercase letter
  89                                  r'[\.\!\?]'          # sentence-ending punct.
  90                                  r'[\"\']?'           # optional end-of-quote
  91                                  % string.lowercase)
  92
  93
  94     def __init__ (self,
  95                   width=70,
  96                   initial_indent="",
  97                   subsequent_indent="",
  98                   expand_tabs=True,
  99                   replace_whitespace=True,
 100                   fix_sentence_endings=False,
 101                   break_long_words=True):
 102         self.width = width
 103         self.initial_indent = initial_indent
 104         self.subsequent_indent = subsequent_indent
 105         self.expand_tabs = expand_tabs
 106         self.replace_whitespace = replace_whitespace
 107         self.fix_sentence_endings = fix_sentence_endings
 108         self.break_long_words = break_long_words
 109
 110
 111     # -- Private methods -----------------------------------------------
 112     # (possibly useful for subclasses to override)
 113
 114     def _munge_whitespace(self, text):
 115         """_munge_whitespace(text : string) -> string
 116
 117         Munge whitespace in text: expand tabs and convert all other
 118         whitespace characters to spaces.  Eg. " foo\tbar\n\nbaz"
 119         becomes " foo    bar  baz".
 120         """
 121         if self.expand_tabs:
 122             text = text.expandtabs()
 123         if self.replace_whitespace:
 124             if isinstance(text, types.StringType):
 125                 text = text.translate(self.whitespace_trans)
 126             elif isinstance(text, types.UnicodeType):
 127                 text = text.translate(self.unicode_whitespace_trans)
 128         return text
 129
 130
 131     def _split(self, text):
 132         """_split(text : string) -> [string]
 133
 134         Split the text to wrap into indivisible chunks.  Chunks are
 135         not quite the same as words; see wrap_chunks() for full
 136         details.  As an example, the text
 137           Look, goof-ball -- use the -b option!
 138         breaks into the following chunks:
 139           'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ',
 140           'use', ' ', 'the', ' ', '-b', ' ', 'option!'
 141         """
 142         chunks = self.wordsep_re.split(text)
 143         chunks = filter(None, chunks)
 144         return chunks
 145
 146     def _fix_sentence_endings(self, chunks):
 147         """_fix_sentence_endings(chunks : [string])
 148
 149         Correct for sentence endings buried in 'chunks'.  Eg. when the
 150         original text contains "... foo.\nBar ...", munge_whitespace()
 151         and split() will convert that to [..., "foo.", " ", "Bar", ...]
 152         which has one too few spaces; this method simply changes the one
 153         space to two.
 154         """
 155         i = 0
 156         pat = self.sentence_end_re
 157         while i < len(chunks)-1:
 158             if chunks[i+1] == " " and pat.search(chunks[i]):
 159                 chunks[i+1] = "  "
 160                 i += 2
 161             else:
 162                 i += 1
 163
 164     def _handle_long_word(self, chunks, cur_line, cur_len, width):
 165         """_handle_long_word(chunks : [string],
 166                              cur_line : [string],
 167                              cur_len : int, width : int)
 168
 169         Handle a chunk of text (most likely a word, not whitespace) that
 170         is too long to fit in any line.
 171         """
 172         space_left = width - cur_len
 173
 174         # If we're allowed to break long words, then do so: put as much
 175         # of the next chunk onto the current line as will fit.
 176         if self.break_long_words:
 177             cur_line.append(chunks[0][0:space_left])
 178             chunks[0] = chunks[0][space_left:]
 179
 180         # Otherwise, we have to preserve the long word intact.  Only add
 181         # it to the current line if there's nothing already there --
 182         # that minimizes how much we violate the width constraint.
 183         elif not cur_line:
 184             cur_line.append(chunks.pop(0))
 185
 186         # If we're not allowed to break long words, and there's already
 187         # text on the current line, do nothing.  Next time through the
 188         # main loop of _wrap_chunks(), we'll wind up here again, but
 189         # cur_len will be zero, so the next line will be entirely
 190         # devoted to the long word that we can't handle right now.
 191
 192     def _wrap_chunks(self, chunks):
 193         """_wrap_chunks(chunks : [string]) -> [string]
 194
 195         Wrap a sequence of text chunks and return a list of lines of
 196         length 'self.width' or less.  (If 'break_long_words' is false,
 197         some lines may be longer than this.)  Chunks correspond roughly
 198         to words and the whitespace between them: each chunk is
 199         indivisible (modulo 'break_long_words'), but a line break can
 200         come between any two chunks.  Chunks should not have internal
 201         whitespace; ie. a chunk is either all whitespace or a "word".
 202         Whitespace chunks will be removed from the beginning and end of
 203         lines, but apart from that whitespace is preserved.
 204         """
 205         lines = []
 206         if self.width <= 0:
 207             raise ValueError("invalid width %r (must be > 0)" % self.width)
 208
 209         while chunks:
 210
 211             # Start the list of chunks that will make up the current line.
 212             # cur_len is just the length of all the chunks in cur_line.
 213             cur_line = []
 214             cur_len = 0
 215
 216             # Figure out which static string will prefix this line.
 217             if lines:
 218                 indent = self.subsequent_indent
 219             else:
 220                 indent = self.initial_indent
 221
 222             # Maximum width for this line.
 223             width = self.width - len(indent)
 224
 225             # First chunk on line is whitespace -- drop it, unless this
 226             # is the very beginning of the text (ie. no lines started yet).
 227             if chunks[0].strip() == '' and lines:
 228                 del chunks[0]
 229
 230             while chunks:
 231                 l = len(chunks[0])
 232
 233                 # Can at least squeeze this chunk onto the current line.
 234                 if cur_len + l <= width:
 235                     cur_line.append(chunks.pop(0))
 236                     cur_len += l
 237
 238                 # Nope, this line is full.
 239                 else:
 240                     break
 241
 242             # The current line is full, and the next chunk is too big to
 243             # fit on *any* line (not just this one).
 244             if chunks and len(chunks[0]) > width:
 245                 self._handle_long_word(chunks, cur_line, cur_len, width)
 246
 247             # If the last chunk on this line is all whitespace, drop it.
 248             if cur_line and cur_line[-1].strip() == '':
 249                 del cur_line[-1]
 250
 251             # Convert current line back to a string and store it in list
 252             # of all lines (return value).
 253             if cur_line:
 254                 lines.append(indent + ''.join(cur_line))
 255
 256         return lines
 257
 258
 259     # -- Public interface ----------------------------------------------
 260
 261     def wrap(self, text):
 262         """wrap(text : string) -> [string]
 263
 264         Reformat the single paragraph in 'text' so it fits in lines of
 265         no more than 'self.width' columns, and return a list of wrapped
 266         lines.  Tabs in 'text' are expanded with string.expandtabs(),
 267         and all other whitespace characters (including newline) are
 268         converted to space.
 269         """
 270         text = self._munge_whitespace(text)
 271         indent = self.initial_indent
 272         if len(text) + len(indent) <= self.width:
 273             return [indent + text]
 274         chunks = self._split(text)
 275         if self.fix_sentence_endings:
 276             self._fix_sentence_endings(chunks)
 277         return self._wrap_chunks(chunks)
 278
 279     def fill(self, text):
 280         """fill(text : string) -> string
 281
 282         Reformat the single paragraph in 'text' to fit in lines of no
 283         more than 'self.width' columns, and return a new string
 284         containing the entire wrapped paragraph.
 285         """
 286         return "\n".join(self.wrap(text))
 287
 288
 289 # -- Convenience interface ---------------------------------------------
 290
 291 def wrap(text, width=70, **kwargs):
 292     """Wrap a single paragraph of text, returning a list of wrapped lines.
 293
 294     Reformat the single paragraph in 'text' so it fits in lines of no
 295     more than 'width' columns, and return a list of wrapped lines.  By
 296     default, tabs in 'text' are expanded with string.expandtabs(), and
 297     all other whitespace characters (including newline) are converted to
 298     space.  See TextWrapper class for available keyword args to customize
 299     wrapping behaviour.
 300     """
 301     w = TextWrapper(width=width, **kwargs)
 302     return w.wrap(text)
 303
 304 def fill(text, width=70, **kwargs):
 305     """Fill a single paragraph of text, returning a new string.
 306
 307     Reformat the single paragraph in 'text' to fit in lines of no more
 308     than 'width' columns, and return a new string containing the entire
 309     wrapped paragraph.  As with wrap(), tabs are expanded and other
 310     whitespace characters converted to space.  See TextWrapper class for
 311     available keyword args to customize wrapping behaviour.
 312     """
 313     w = TextWrapper(width=width, **kwargs)
 314     return w.fill(text)
 315
 316
 317 # -- Loosely related functionality -------------------------------------
 318
 319 def dedent(text):
 320     """dedent(text : string) -> string
 321
 322     Remove any whitespace than can be uniformly removed from the left
 323     of every line in `text`.
 324
 325     This can be used e.g. to make triple-quoted strings line up with
 326     the left edge of screen/whatever, while still presenting it in the
 327     source code in indented form.
 328
 329     For example:
 330
 331         def test():
 332             # end first line with \ to avoid the empty line!
 333             s = '''\
 334             hello
 335               world
 336             '''
 337             print repr(s)          # prints '    hello\n      world\n    '
 338             print repr(dedent(s))  # prints 'hello\n  world\n'
 339     """
 340     lines = text.expandtabs().split('\n')
 341     margin = None
 342     for line in lines:
 343         content = line.lstrip()
 344         if not content:
 345             continue
 346         indent = len(line) - len(content)
 347         if margin is None:
 348             margin = indent
 349         else:
 350             margin = min(margin, indent)
 351
 352     if margin is not None and margin > 0:
 353         for i in range(len(lines)):
 354             lines[i] = lines[i][margin:]
 355
 356     return '\n'.join(lines)