Lib/textwrap.py

   1 """Text wrapping and filling.
   2 """
   3
   4 # Copyright (C) 1999-2001 Gregory P. Ward.
   5 # Copyright (C) 2002, 2003 Python Software Foundation.
   6 # Written by Greg Ward <gward@python.net>
   7
   8 __revision__ = "$Id$"
   9
  10 import string, re
  11
  12 # Do the right thing with boolean values for all known Python versions
  13 # (so this module can be copied to projects that don't depend on Python
  14 # 2.3, e.g. Optik and Docutils) by uncommenting the block of code below.
  15 #try:
  16 #    True, False
  17 #except NameError:
  18 #    (True, False) = (1, 0)
  19
  20 __all__ = ['TextWrapper', 'wrap', 'fill', 'dedent']
  21
  22 # Hardcode the recognized whitespace characters to the US-ASCII
  23 # whitespace characters.  The main reason for doing this is that in
  24 # ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales
  25 # that character winds up in string.whitespace.  Respecting
  26 # string.whitespace in those cases would 1) make textwrap treat 0xa0 the
  27 # same as any other whitespace char, which is clearly wrong (it's a
  28 # *non-breaking* space), 2) possibly cause problems with Unicode,
  29 # since 0xa0 is not in range(128).
  30 _whitespace = '\t\n\x0b\x0c\r '
  31
  32 class TextWrapper:
  33     """
  34     Object for wrapping/filling text.  The public interface consists of
  35     the wrap() and fill() methods; the other methods are just there for
  36     subclasses to override in order to tweak the default behaviour.
  37     If you want to completely replace the main wrapping algorithm,
  38     you'll probably have to override _wrap_chunks().
  39
  40     Several instance attributes control various aspects of wrapping:
  41       width (default: 70)
  42         the maximum width of wrapped lines (unless break_long_words
  43         is false)
  44       initial_indent (default: "")
  45         string that will be prepended to the first line of wrapped
  46         output.  Counts towards the line's width.
  47       subsequent_indent (default: "")
  48         string that will be prepended to all lines save the first
  49         of wrapped output; also counts towards each line's width.
  50       expand_tabs (default: true)
  51         Expand tabs in input text to spaces before further processing.
  52         Each tab will become 1 .. 8 spaces, depending on its position in
  53         its line.  If false, each tab is treated as a single character.
  54       replace_whitespace (default: true)
  55         Replace all whitespace characters in the input text by spaces
  56         after tab expansion.  Note that if expand_tabs is false and
  57         replace_whitespace is true, every tab will be converted to a
  58         single space!
  59       fix_sentence_endings (default: false)
  60         Ensure that sentence-ending punctuation is always followed
  61         by two spaces.  Off by default because the algorithm is
  62         (unavoidably) imperfect.
  63       break_long_words (default: true)
  64         Break words longer than 'width'.  If false, those words will not
  65         be broken, and some lines might be longer than 'width'.
  66       break_on_hyphens (default: true)
  67         Allow breaking hyphenated words. If true, wrapping will occur
  68         preferably on whitespaces and right after hyphens part of
  69         compound words.
  70       drop_whitespace (default: true)
  71         Drop leading and trailing whitespace from lines.
  72     """
  73
  74     whitespace_trans = string.maketrans(_whitespace, ' ' * len(_whitespace))
  75
  76     unicode_whitespace_trans = {}
  77     uspace = ord(u' ')
  78     for x in map(ord, _whitespace):
  79         unicode_whitespace_trans[x] = uspace
  80
  81     # This funky little regex is just the trick for splitting
  82     # text up into word-wrappable chunks.  E.g.
  83     #   "Hello there -- you goof-ball, use the -b option!"
  84     # splits into
  85     #   Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option!
  86     # (after stripping out empty strings).
  87     wordsep_re = re.compile(
  88         r'(\s+|'                                  # any whitespace
  89         r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|'   # hyphenated words
  90         r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))')   # em-dash
  91
  92     # This less funky little regex just split on recognized spaces. E.g.
  93     #   "Hello there -- you goof-ball, use the -b option!"
  94     # splits into
  95     #   Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/
  96     wordsep_simple_re = re.compile(r'(\s+)')
  97
  98     # XXX this is not locale- or charset-aware -- string.lowercase
  99     # is US-ASCII only (and therefore English-only)
 100     sentence_end_re = re.compile(r'[%s]'              # lowercase letter
 101                                  r'[\.\!\?]'          # sentence-ending punct.
 102                                  r'[\"\']?'           # optional end-of-quote
 103                                  r'\Z'                # end of chunk
 104                                  % string.lowercase)
 105
 106
 107     def __init__(self,
 108                  width=70,
 109                  initial_indent="",
 110                  subsequent_indent="",
 111                  expand_tabs=True,
 112                  replace_whitespace=True,
 113                  fix_sentence_endings=False,
 114                  break_long_words=True,
 115                  drop_whitespace=True,
 116                  break_on_hyphens=True):
 117         self.width = width
 118         self.initial_indent = initial_indent
 119         self.subsequent_indent = subsequent_indent
 120         self.expand_tabs = expand_tabs
 121         self.replace_whitespace = replace_whitespace
 122         self.fix_sentence_endings = fix_sentence_endings
 123         self.break_long_words = break_long_words
 124         self.drop_whitespace = drop_whitespace
 125         self.break_on_hyphens = break_on_hyphens
 126
 127         # recompile the regexes for Unicode mode -- done in this clumsy way for
 128         # backwards compatibility because it's rather common to monkey-patch
 129         # the TextWrapper class' wordsep_re attribute.
 130         self.wordsep_re_uni = re.compile(self.wordsep_re.pattern, re.U)
 131         self.wordsep_simple_re_uni = re.compile(
 132             self.wordsep_simple_re.pattern, re.U)
 133
 134
 135     # -- Private methods -----------------------------------------------
 136     # (possibly useful for subclasses to override)
 137
 138     def _munge_whitespace(self, text):
 139         """_munge_whitespace(text : string) -> string
 140
 141         Munge whitespace in text: expand tabs and convert all other
 142         whitespace characters to spaces.  Eg. " foo\tbar\n\nbaz"
 143         becomes " foo    bar  baz".
 144         """
 145         if self.expand_tabs:
 146             text = text.expandtabs()
 147         if self.replace_whitespace:
 148             if isinstance(text, str):
 149                 text = text.translate(self.whitespace_trans)
 150             elif isinstance(text, unicode):
 151                 text = text.translate(self.unicode_whitespace_trans)
 152         return text
 153
 154
 155     def _split(self, text):
 156         """_split(text : string) -> [string]
 157
 158         Split the text to wrap into indivisible chunks.  Chunks are
 159         not quite the same as words; see wrap_chunks() for full
 160         details.  As an example, the text
 161           Look, goof-ball -- use the -b option!
 162         breaks into the following chunks:
 163           'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ',
 164           'use', ' ', 'the', ' ', '-b', ' ', 'option!'
 165         if break_on_hyphens is True, or in:
 166           'Look,', ' ', 'goof-ball', ' ', '--', ' ',
 167           'use', ' ', 'the', ' ', '-b', ' ', option!'
 168         otherwise.
 169         """
 170         if isinstance(text, unicode):
 171             if self.break_on_hyphens:
 172                 pat = self.wordsep_re_uni
 173             else:
 174                 pat = self.wordsep_simple_re_uni
 175         else:
 176             if self.break_on_hyphens:
 177                 pat = self.wordsep_re
 178             else:
 179                 pat = self.wordsep_simple_re
 180         chunks = pat.split(text)
 181         chunks = filter(None, chunks)  # remove empty chunks
 182         return chunks
 183
 184     def _fix_sentence_endings(self, chunks):
 185         """_fix_sentence_endings(chunks : [string])
 186
 187         Correct for sentence endings buried in 'chunks'.  Eg. when the
 188         original text contains "... foo.\nBar ...", munge_whitespace()
 189         and split() will convert that to [..., "foo.", " ", "Bar", ...]
 190         which has one too few spaces; this method simply changes the one
 191         space to two.
 192         """
 193         i = 0
 194         pat = self.sentence_end_re
 195         while i < len(chunks)-1:
 196             if chunks[i+1] == " " and pat.search(chunks[i]):
 197                 chunks[i+1] = "  "
 198                 i += 2
 199             else:
 200                 i += 1
 201
 202     def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
 203         """_handle_long_word(chunks : [string],
 204                              cur_line : [string],
 205                              cur_len : int, width : int)
 206
 207         Handle a chunk of text (most likely a word, not whitespace) that
 208         is too long to fit in any line.
 209         """
 210         # Figure out when indent is larger than the specified width, and make
 211         # sure at least one character is stripped off on every pass
 212         if width < 1:
 213             space_left = 1
 214         else:
 215             space_left = width - cur_len
 216
 217         # If we're allowed to break long words, then do so: put as much
 218         # of the next chunk onto the current line as will fit.
 219         if self.break_long_words:
 220             cur_line.append(reversed_chunks[-1][:space_left])
 221             reversed_chunks[-1] = reversed_chunks[-1][space_left:]
 222
 223         # Otherwise, we have to preserve the long word intact.  Only add
 224         # it to the current line if there's nothing already there --
 225         # that minimizes how much we violate the width constraint.
 226         elif not cur_line:
 227             cur_line.append(reversed_chunks.pop())
 228
 229         # If we're not allowed to break long words, and there's already
 230         # text on the current line, do nothing.  Next time through the
 231         # main loop of _wrap_chunks(), we'll wind up here again, but
 232         # cur_len will be zero, so the next line will be entirely
 233         # devoted to the long word that we can't handle right now.
 234
 235     def _wrap_chunks(self, chunks):
 236         """_wrap_chunks(chunks : [string]) -> [string]
 237
 238         Wrap a sequence of text chunks and return a list of lines of
 239         length 'self.width' or less.  (If 'break_long_words' is false,
 240         some lines may be longer than this.)  Chunks correspond roughly
 241         to words and the whitespace between them: each chunk is
 242         indivisible (modulo 'break_long_words'), but a line break can
 243         come between any two chunks.  Chunks should not have internal
 244         whitespace; ie. a chunk is either all whitespace or a "word".
 245         Whitespace chunks will be removed from the beginning and end of
 246         lines, but apart from that whitespace is preserved.
 247         """
 248         lines = []
 249         if self.width <= 0:
 250             raise ValueError("invalid width %r (must be > 0)" % self.width)
 251
 252         # Arrange in reverse order so items can be efficiently popped
 253         # from a stack of chucks.
 254         chunks.reverse()
 255
 256         while chunks:
 257
 258             # Start the list of chunks that will make up the current line.
 259             # cur_len is just the length of all the chunks in cur_line.
 260             cur_line = []
 261             cur_len = 0
 262
 263             # Figure out which static string will prefix this line.
 264             if lines:
 265                 indent = self.subsequent_indent
 266             else:
 267                 indent = self.initial_indent
 268
 269             # Maximum width for this line.
 270             width = self.width - len(indent)
 271
 272             # First chunk on line is whitespace -- drop it, unless this
 273             # is the very beginning of the text (ie. no lines started yet).
 274             if self.drop_whitespace and chunks[-1].strip() == '' and lines:
 275                 del chunks[-1]
 276
 277             while chunks:
 278                 l = len(chunks[-1])
 279
 280                 # Can at least squeeze this chunk onto the current line.
 281                 if cur_len + l <= width:
 282                     cur_line.append(chunks.pop())
 283                     cur_len += l
 284
 285                 # Nope, this line is full.
 286                 else:
 287                     break
 288
 289             # The current line is full, and the next chunk is too big to
 290             # fit on *any* line (not just this one).
 291             if chunks and len(chunks[-1]) > width:
 292                 self._handle_long_word(chunks, cur_line, cur_len, width)
 293
 294             # If the last chunk on this line is all whitespace, drop it.
 295             if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':
 296                 del cur_line[-1]
 297
 298             # Convert current line back to a string and store it in list
 299             # of all lines (return value).
 300             if cur_line:
 301                 lines.append(indent + ''.join(cur_line))
 302
 303         return lines
 304
 305
 306     # -- Public interface ----------------------------------------------
 307
 308     def wrap(self, text):
 309         """wrap(text : string) -> [string]
 310
 311         Reformat the single paragraph in 'text' so it fits in lines of
 312         no more than 'self.width' columns, and return a list of wrapped
 313         lines.  Tabs in 'text' are expanded with string.expandtabs(),
 314         and all other whitespace characters (including newline) are
 315         converted to space.
 316         """
 317         text = self._munge_whitespace(text)
 318         chunks = self._split(text)
 319         if self.fix_sentence_endings:
 320             self._fix_sentence_endings(chunks)
 321         return self._wrap_chunks(chunks)
 322
 323     def fill(self, text):
 324         """fill(text : string) -> string
 325
 326         Reformat the single paragraph in 'text' to fit in lines of no
 327         more than 'self.width' columns, and return a new string
 328         containing the entire wrapped paragraph.
 329         """
 330         return "\n".join(self.wrap(text))
 331
 332
 333 # -- Convenience interface ---------------------------------------------
 334
 335 def wrap(text, width=70, **kwargs):
 336     """Wrap a single paragraph of text, returning a list of wrapped lines.
 337
 338     Reformat the single paragraph in 'text' so it fits in lines of no
 339     more than 'width' columns, and return a list of wrapped lines.  By
 340     default, tabs in 'text' are expanded with string.expandtabs(), and
 341     all other whitespace characters (including newline) are converted to
 342     space.  See TextWrapper class for available keyword args to customize
 343     wrapping behaviour.
 344     """
 345     w = TextWrapper(width=width, **kwargs)
 346     return w.wrap(text)
 347
 348 def fill(text, width=70, **kwargs):
 349     """Fill a single paragraph of text, returning a new string.
 350
 351     Reformat the single paragraph in 'text' to fit in lines of no more
 352     than 'width' columns, and return a new string containing the entire
 353     wrapped paragraph.  As with wrap(), tabs are expanded and other
 354     whitespace characters converted to space.  See TextWrapper class for
 355     available keyword args to customize wrapping behaviour.
 356     """
 357     w = TextWrapper(width=width, **kwargs)
 358     return w.fill(text)
 359
 360
 361 # -- Loosely related functionality -------------------------------------
 362
 363 _whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE)
 364 _leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE)
 365
 366 def dedent(text):
 367     """Remove any common leading whitespace from every line in `text`.
 368
 369     This can be used to make triple-quoted strings line up with the left
 370     edge of the display, while still presenting them in the source code
 371     in indented form.
 372
 373     Note that tabs and spaces are both treated as whitespace, but they
 374     are not equal: the lines "  hello" and "\thello" are
 375     considered to have no common leading whitespace.  (This behaviour is
 376     new in Python 2.5; older versions of this module incorrectly
 377     expanded tabs before searching for common leading whitespace.)
 378     """
 379     # Look for the longest leading string of spaces and tabs common to
 380     # all lines.
 381     margin = None
 382     text = _whitespace_only_re.sub('', text)
 383     indents = _leading_whitespace_re.findall(text)
 384     for indent in indents:
 385         if margin is None:
 386             margin = indent
 387
 388         # Current line more deeply indented than previous winner:
 389         # no change (previous winner is still on top).
 390         elif indent.startswith(margin):
 391             pass
 392
 393         # Current line consistent with and no deeper than previous winner:
 394         # it's the new winner.
 395         elif margin.startswith(indent):
 396             margin = indent
 397
 398         # Current line and previous winner have no common whitespace:
 399         # there is no margin.
 400         else:
 401             margin = ""
 402             break
 403
 404     # sanity check (testing/debugging only)
 405     if 0 and margin:
 406         for line in text.split("\n"):
 407             assert not line or line.startswith(margin), \
 408                    "line = %r, margin = %r" % (line, margin)
 409
 410     if margin:
 411         text = re.sub(r'(?m)^' + margin, '', text)
 412     return text
 413
 414 if __name__ == "__main__":
 415     #print dedent("\tfoo\n\tbar")
 416     #print dedent("  \thello there\n  \t  how are you?")
 417     print dedent("Hello there.\n  This is indented.")