docutils/docutils/utils/smartquotes.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 # :Id: $Id$
   5 # :Copyright: © 2010 Günter Milde,
   6 #             original `SmartyPants`_: © 2003 John Gruber
   7 #             smartypants.py:          © 2004, 2007 Chad Miller
   8 # :Maintainer: docutils-develop@lists.sourceforge.net
   9 # :License: Released under the terms of the `2-Clause BSD license`_, in short:
  10 #
  11 #    Copying and distribution of this file, with or without modification,
  12 #    are permitted in any medium without royalty provided the copyright
  13 #    notices and this notice are preserved.
  14 #    This file is offered as-is, without any warranty.
  15 #
  16 # .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause
  17
  18
  19 r"""
  20 ========================
  21 SmartyPants for Docutils
  22 ========================
  23
  24 Synopsis
  25 ========
  26
  27 Smart-quotes for Docutils.
  28
  29 The original "SmartyPants" is a free web publishing plug-in for Movable Type,
  30 Blosxom, and BBEdit that easily translates plain ASCII punctuation characters
  31 into "smart" typographic punctuation characters.
  32
  33 `smartypants.py`, endeavours to be a functional port of
  34 SmartyPants to Python, for use with Pyblosxom_.
  35
  36 `smartquotes.py` is an adaption of Smartypants to Docutils_. By using Unicode
  37 characters instead of HTML entities for typographic quotes, it works for any
  38 output format that supports Unicode.
  39
  40 Authors
  41 =======
  42
  43 `John Gruber`_ did all of the hard work of writing this software in Perl for
  44 `Movable Type`_ and almost all of this useful documentation.  `Chad Miller`_
  45 ported it to Python to use with Pyblosxom_.
  46 Adapted to Docutils_ by Günter Milde
  47
  48 Additional Credits
  49 ==================
  50
  51 Portions of the SmartyPants original work are based on Brad Choate's nifty
  52 MTRegex plug-in.  `Brad Choate`_ also contributed a few bits of source code to
  53 this plug-in.  Brad Choate is a fine hacker indeed.
  54
  55 `Jeremy Hedley`_ and `Charles Wiltgen`_ deserve mention for exemplary beta
  56 testing of the original SmartyPants.
  57
  58 `Rael Dornfest`_ ported SmartyPants to Blosxom.
  59
  60 .. _Brad Choate: http://bradchoate.com/
  61 .. _Jeremy Hedley: http://antipixel.com/
  62 .. _Charles Wiltgen: http://playbacktime.com/
  63 .. _Rael Dornfest: http://raelity.org/
  64
  65
  66 Copyright and License
  67 =====================
  68
  69 SmartyPants_ license (3-Clause BSD license):
  70
  71   Copyright (c) 2003 John Gruber (http://daringfireball.net/)
  72   All rights reserved.
  73
  74   Redistribution and use in source and binary forms, with or without
  75   modification, are permitted provided that the following conditions are
  76   met:
  77
  78   * Redistributions of source code must retain the above copyright
  79     notice, this list of conditions and the following disclaimer.
  80
  81   * Redistributions in binary form must reproduce the above copyright
  82     notice, this list of conditions and the following disclaimer in
  83     the documentation and/or other materials provided with the
  84     distribution.
  85
  86   * Neither the name "SmartyPants" nor the names of its contributors
  87     may be used to endorse or promote products derived from this
  88     software without specific prior written permission.
  89
  90   This software is provided by the copyright holders and contributors
  91   "as is" and any express or implied warranties, including, but not
  92   limited to, the implied warranties of merchantability and fitness for
  93   a particular purpose are disclaimed. In no event shall the copyright
  94   owner or contributors be liable for any direct, indirect, incidental,
  95   special, exemplary, or consequential damages (including, but not
  96   limited to, procurement of substitute goods or services; loss of use,
  97   data, or profits; or business interruption) however caused and on any
  98   theory of liability, whether in contract, strict liability, or tort
  99   (including negligence or otherwise) arising in any way out of the use
 100   of this software, even if advised of the possibility of such damage.
 101
 102 smartypants.py license (2-Clause BSD license):
 103
 104   smartypants.py is a derivative work of SmartyPants.
 105
 106   Redistribution and use in source and binary forms, with or without
 107   modification, are permitted provided that the following conditions are
 108   met:
 109
 110   * Redistributions of source code must retain the above copyright
 111     notice, this list of conditions and the following disclaimer.
 112
 113   * Redistributions in binary form must reproduce the above copyright
 114     notice, this list of conditions and the following disclaimer in
 115     the documentation and/or other materials provided with the
 116     distribution.
 117
 118   This software is provided by the copyright holders and contributors
 119   "as is" and any express or implied warranties, including, but not
 120   limited to, the implied warranties of merchantability and fitness for
 121   a particular purpose are disclaimed. In no event shall the copyright
 122   owner or contributors be liable for any direct, indirect, incidental,
 123   special, exemplary, or consequential damages (including, but not
 124   limited to, procurement of substitute goods or services; loss of use,
 125   data, or profits; or business interruption) however caused and on any
 126   theory of liability, whether in contract, strict liability, or tort
 127   (including negligence or otherwise) arising in any way out of the use
 128   of this software, even if advised of the possibility of such damage.
 129
 130 .. _John Gruber: http://daringfireball.net/
 131 .. _Chad Miller: http://web.chad.org/
 132
 133 .. _Pyblosxom: http://pyblosxom.bluesock.org/
 134 .. _SmartyPants: http://daringfireball.net/projects/smartypants/
 135 .. _Movable Type: http://www.movabletype.org/
 136 .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause
 137 .. _Docutils: http://docutils.sf.net/
 138
 139 Description
 140 ===========
 141
 142 SmartyPants can perform the following transformations:
 143
 144 - Straight quotes ( " and ' ) into "curly" quote characters
 145 - Backticks-style quotes (\`\`like this'') into "curly" quote characters
 146 - Dashes (``--`` and ``---``) into en- and em-dash entities
 147 - Three consecutive dots (``...`` or ``. . .``) into an ellipsis entity
 148
 149 This means you can write, edit, and save your posts using plain old
 150 ASCII straight quotes, plain dashes, and plain dots, but your published
 151 posts (and final HTML output) will appear with smart quotes, em-dashes,
 152 and proper ellipses.
 153
 154 SmartyPants does not modify characters within ``<pre>``, ``<code>``, ``<kbd>``,
 155 ``<math>`` or ``<script>`` tag blocks. Typically, these tags are used to
 156 display text where smart quotes and other "smart punctuation" would not be
 157 appropriate, such as source code or example markup.
 158
 159
 160 Backslash Escapes
 161 =================
 162
 163 If you need to use literal straight quotes (or plain hyphens and
 164 periods), SmartyPants accepts the following backslash escape sequences
 165 to force non-smart punctuation. It does so by transforming the escape
 166 sequence into a character:
 167
 168 ========  =====  =========
 169 Escape    Value  Character
 170 ========  =====  =========
 171 ``\\\\``  &#92;  \\
 172 \\"       &#34;  "
 173 \\'       &#39;  '
 174 \\.       &#46;  .
 175 \\-       &#45;  \-
 176 \\`       &#96;  \`
 177 ========  =====  =========
 178
 179 This is useful, for example, when you want to use straight quotes as
 180 foot and inch marks: 6\\'2\\" tall; a 17\\" iMac.
 181
 182 Options
 183 =======
 184
 185 For Pyblosxom users, the ``smartypants_attributes`` attribute is where you
 186 specify configuration options.
 187
 188 Numeric values are the easiest way to configure SmartyPants' behavior:
 189
 190 "0"
 191         Suppress all transformations. (Do nothing.)
 192 "1"
 193         Performs default SmartyPants transformations: quotes (including
 194         \`\`backticks'' -style), em-dashes, and ellipses. "``--``" (dash dash)
 195         is used to signify an em-dash; there is no support for en-dashes.
 196
 197 "2"
 198         Same as smarty_pants="1", except that it uses the old-school typewriter
 199         shorthand for dashes:  "``--``" (dash dash) for en-dashes, "``---``"
 200         (dash dash dash)
 201         for em-dashes.
 202
 203 "3"
 204         Same as smarty_pants="2", but inverts the shorthand for dashes:
 205         "``--``" (dash dash) for em-dashes, and "``---``" (dash dash dash) for
 206         en-dashes.
 207
 208 "-1"
 209         Stupefy mode. Reverses the SmartyPants transformation process, turning
 210         the characters produced by SmartyPants into their ASCII equivalents.
 211         E.g.  "“" is turned into a simple double-quote (\"), "—" is
 212         turned into two dashes, etc.
 213
 214
 215 The following single-character attribute values can be combined to toggle
 216 individual transformations from within the smarty_pants attribute. For
 217 example, to educate normal quotes and em-dashes, but not ellipses or
 218 \`\`backticks'' -style quotes:
 219
 220 ``py['smartypants_attributes'] = "1"``
 221
 222 "q"
 223         Educates normal quote characters: (") and (').
 224
 225 "b"
 226         Educates \`\`backticks'' -style double quotes.
 227
 228 "B"
 229         Educates \`\`backticks'' -style double quotes and \`single' quotes.
 230
 231 "d"
 232         Educates em-dashes.
 233
 234 "D"
 235         Educates em-dashes and en-dashes, using old-school typewriter shorthand:
 236         (dash dash) for en-dashes, (dash dash dash) for em-dashes.
 237
 238 "i"
 239         Educates em-dashes and en-dashes, using inverted old-school typewriter
 240         shorthand: (dash dash) for em-dashes, (dash dash dash) for en-dashes.
 241
 242 "e"
 243         Educates ellipses.
 244
 245 "w"
 246         Translates any instance of ``&quot;`` into a normal double-quote character.
 247         This should be of no interest to most people, but of particular interest
 248         to anyone who writes their posts using Dreamweaver, as Dreamweaver
 249         inexplicably uses this entity to represent a literal double-quote
 250         character. SmartyPants only educates normal quotes, not entities (because
 251         ordinarily, entities are used for the explicit purpose of representing the
 252         specific character they represent). The "w" option must be used in
 253         conjunction with one (or both) of the other quote options ("q" or "b").
 254         Thus, if you wish to apply all SmartyPants transformations (quotes, en-
 255         and em-dashes, and ellipses) and also translate ``&quot;`` entities into
 256         regular quotes so SmartyPants can educate them, you should pass the
 257         following to the smarty_pants attribute:
 258
 259
 260 Caveats
 261 =======
 262
 263 Why You Might Not Want to Use Smart Quotes in Your Weblog
 264 ---------------------------------------------------------
 265
 266 For one thing, you might not care.
 267
 268 Most normal, mentally stable individuals do not take notice of proper
 269 typographic punctuation. Many design and typography nerds, however, break
 270 out in a nasty rash when they encounter, say, a restaurant sign that uses
 271 a straight apostrophe to spell "Joe's".
 272
 273 If you're the sort of person who just doesn't care, you might well want to
 274 continue not caring. Using straight quotes -- and sticking to the 7-bit
 275 ASCII character set in general -- is certainly a simpler way to live.
 276
 277 Even if you I *do* care about accurate typography, you still might want to
 278 think twice before educating the quote characters in your weblog. One side
 279 effect of publishing curly quote characters is that it makes your
 280 weblog a bit harder for others to quote from using copy-and-paste. What
 281 happens is that when someone copies text from your blog, the copied text
 282 contains the 8-bit curly quote characters (as well as the 8-bit characters
 283 for em-dashes and ellipses, if you use these options). These characters
 284 are not standard across different text encoding methods, which is why they
 285 need to be encoded as characters.
 286
 287 People copying text from your weblog, however, may not notice that you're
 288 using curly quotes, and they'll go ahead and paste the unencoded 8-bit
 289 characters copied from their browser into an email message or their own
 290 weblog. When pasted as raw "smart quotes", these characters are likely to
 291 get mangled beyond recognition.
 292
 293 That said, my own opinion is that any decent text editor or email client
 294 makes it easy to stupefy smart quote characters into their 7-bit
 295 equivalents, and I don't consider it my problem if you're using an
 296 indecent text editor or email client.
 297
 298
 299 Algorithmic Shortcomings
 300 ------------------------
 301
 302 One situation in which quotes will get curled the wrong way is when
 303 apostrophes are used at the start of leading contractions. For example:
 304
 305 ``'Twas the night before Christmas.``
 306
 307 In the case above, SmartyPants will turn the apostrophe into an opening
 308 single-quote, when in fact it should be a closing one. I don't think
 309 this problem can be solved in the general case -- every word processor
 310 I've tried gets this wrong as well. In such cases, it's best to use the
 311 proper character for closing single-quotes (``’``) by hand.
 312
 313
 314 Version History
 315 ===============
 316
 317 1.7     2012-11-19
 318         - Internationalization: language-dependent quotes.
 319
 320 1.6.1:  2012-11-06
 321         - Refactor code, code cleanup,
 322         - `educate_tokens()` generator as interface for Docutils.
 323
 324 1.6:    2010-08-26
 325         - Adaption to Docutils:
 326           - Use Unicode instead of HTML entities,
 327           - Remove code special to pyblosxom.
 328
 329 1.5_1.6: Fri, 27 Jul 2007 07:06:40 -0400
 330         - Fixed bug where blocks of precious unalterable text was instead
 331           interpreted.  Thanks to Le Roux and Dirk van Oosterbosch.
 332
 333 1.5_1.5: Sat, 13 Aug 2005 15:50:24 -0400
 334         - Fix bogus magical quotation when there is no hint that the
 335           user wants it, e.g., in "21st century".  Thanks to Nathan Hamblen.
 336         - Be smarter about quotes before terminating numbers in an en-dash'ed
 337           range.
 338
 339 1.5_1.4: Thu, 10 Feb 2005 20:24:36 -0500
 340         - Fix a date-processing bug, as reported by jacob childress.
 341         - Begin a test-suite for ensuring correct output.
 342         - Removed import of "string", since I didn't really need it.
 343           (This was my first every Python program.  Sue me!)
 344
 345 1.5_1.3: Wed, 15 Sep 2004 18:25:58 -0400
 346         - Abort processing if the flavour is in forbidden-list.  Default of
 347           [ "rss" ]   (Idea of Wolfgang SCHNERRING.)
 348         - Remove stray virgules from en-dashes.  Patch by Wolfgang SCHNERRING.
 349
 350 1.5_1.2: Mon, 24 May 2004 08:14:54 -0400
 351         - Some single quotes weren't replaced properly.  Diff-tesuji played
 352           by Benjamin GEIGER.
 353
 354 1.5_1.1: Sun, 14 Mar 2004 14:38:28 -0500
 355         - Support upcoming pyblosxom 0.9 plugin verification feature.
 356
 357 1.5_1.0: Tue, 09 Mar 2004 08:08:35 -0500
 358         - Initial release
 359 """
 360
 361 default_smartypants_attr = "1"
 362
 363
 364 import re
 365
 366 class smartchars(object):
 367     """Smart quotes and dashes
 368     """
 369
 370     endash   = u'–' # "&#8211;" EN DASH
 371     emdash   = u'—' # "&#8212;" EM DASH
 372     ellipsis = u'…' # "&#8230;" HORIZONTAL ELLIPSIS
 373
 374     # quote characters (language-specific, set in __init__())
 375     #
 376     # English smart quotes (open primary, close primary, open secondary, close
 377     # secondary) are:
 378     #   opquote  = u'“' # "&#8220;" LEFT DOUBLE QUOTATION MARK
 379     #   cpquote  = u'”' # "&#8221;" RIGHT DOUBLE QUOTATION MARK
 380     #   osquote  = u'‘' # "&#8216;" LEFT SINGLE QUOTATION MARK
 381     #   csquote  = u'’' # "&#8217;" RIGHT SINGLE QUOTATION MARK
 382     # For other languages see:
 383     # http://en.wikipedia.org/wiki/Non-English_usage_of_quotation_marks
 384     # http://de.wikipedia.org/wiki/Anf%C3%BChrungszeichen#Andere_Sprachen
 385     quotes = {'af':           u'“”‘’',
 386               'af-x-altquot': u'„”‚’',
 387               'ca':           u'«»“”',
 388               'ca-x-altquot': u'“”‘’',
 389               'cs':           u'„“‚‘',
 390               'cs-x-altquot': u'»«›‹',
 391               'de':           u'„“‚‘',
 392               'de-x-altquot': u'»«›‹',
 393               'de-CH':        u'«»‹›',
 394               'el':           u'«»“”',
 395               'en':           u'“”‘’',
 396               'en-UK':        u'‘’“”',
 397               'eo':           u'“”‘’',
 398               'es':           u'«»“”',
 399               'et':           u'„“‚‘', # no secondary quote listed in
 400               'et-x-altquot': u'»«›‹', # the sources above (wikipedia.org)
 401               'eu':           u'«»‹›',
 402               'es-x-altquot': u'“”‘’',
 403               'fi':           u'””’’',
 404               'fi-x-altquot': u'»»’’',
 405               'fr':           (u'« ',  u' »', u'‹ ', u' ›'), # with narrow no-break space
 406               'fr-x-altquot': u'«»‹›', # for use with manually set spaces
 407               # 'fr-x-altquot': (u'“ ',  u' ”', u'‘ ', u' ’'), # rarely used
 408               'fr-CH':        u'«»‹›',
 409               'gl':           u'«»“”',
 410               'he':           u'”“»«',
 411               'he-x-altquot': u'„”‚’',
 412               'it':           u'«»“”',
 413               'it-CH':        u'«»‹›',
 414               'it-x-altquot': u'“”‘’',
 415               'ja':           u'「」『』',
 416               'lt':           u'„“‚‘',
 417               'nl':           u'“”‘’',
 418               'nl-x-altquot': u'„”‚’',
 419               'pl':           u'„”«»',
 420               'pl-x-altquot': u'«»“”',
 421               'pt':           u'«»“”',
 422               'pt-BR':        u'“”‘’',
 423               'ro':           u'„”«»',
 424               'ro-x-altquot': u'«»„”',
 425               'ru':           u'«»„“',
 426               'sk':           u'„“‚‘',
 427               'sk-x-altquot': u'»«›‹',
 428               'sv':           u'„“‚‘',
 429               'sv-x-altquot': u'»«›‹',
 430               'zh-CN':        u'“”‘’',
 431               'it':           u'«»“”',
 432               'zh-TW':        u'「」『』',
 433              }
 434
 435     def __init__(self, language='en'):
 436         self.language = language
 437         try:
 438             (self.opquote, self.cpquote,
 439              self.osquote, self.csquote) = self.quotes[language]
 440         except KeyError:
 441             self.opquote, self.cpquote, self.osquote, self.csquote = u'""\'\''
 442
 443
 444 def smartyPants(text, attr=default_smartypants_attr, language='en'):
 445     """Main function for "traditional" use."""
 446
 447     return "".join([t for t in educate_tokens(tokenize(text),
 448                                               attr, language)])
 449
 450
 451 def educate_tokens(text_tokens, attr=default_smartypants_attr, language='en'):
 452     """Return iterator that "educates" the items of `text_tokens`.
 453     """
 454
 455     # Parse attributes:
 456     # 0 : do nothing
 457     # 1 : set all
 458     # 2 : set all, using old school en- and em- dash shortcuts
 459     # 3 : set all, using inverted old school en and em- dash shortcuts
 460     #
 461     # q : quotes
 462     # b : backtick quotes (``double'' only)
 463     # B : backtick quotes (``double'' and `single')
 464     # d : dashes
 465     # D : old school dashes
 466     # i : inverted old school dashes
 467     # e : ellipses
 468     # w : convert &quot; entities to " for Dreamweaver users
 469
 470     convert_quot = False  # translate &quot; entities into normal quotes?
 471     do_dashes = False
 472     do_backticks = False
 473     do_quotes = False
 474     do_ellipses = False
 475     do_stupefy = False
 476
 477     if attr == "0": # Do nothing.
 478         yield text
 479     elif attr == "1": # Do everything, turn all options on.
 480         do_quotes    = True
 481         do_backticks = True
 482         do_dashes    = 1
 483         do_ellipses  = True
 484     elif attr == "2":
 485         # Do everything, turn all options on, use old school dash shorthand.
 486         do_quotes    = True
 487         do_backticks = True
 488         do_dashes    = 2
 489         do_ellipses  = True
 490     elif attr == "3":
 491         # Do everything, use inverted old school dash shorthand.
 492         do_quotes    = True
 493         do_backticks = True
 494         do_dashes    = 3
 495         do_ellipses  = True
 496     elif attr == "-1": # Special "stupefy" mode.
 497         do_stupefy   = True
 498     else:
 499         if "q" in attr: do_quotes = True
 500         if "b" in attr: do_backticks = True
 501         if "B" in attr: do_backticks = 2
 502         if "d" in attr: do_dashes = 1
 503         if "D" in attr: do_dashes = 2
 504         if "i" in attr: do_dashes = 3
 505         if "e" in attr: do_ellipses = True
 506         if "w" in attr: convert_quot = True
 507
 508     prev_token_last_char = " "
 509     # Last character of the previous text token. Used as
 510     # context to curl leading quote characters correctly.
 511
 512     for (ttype, text) in text_tokens:
 513
 514         # skip HTML and/or XML tags as well as emtpy text tokens
 515         # without updating the last character
 516         if ttype == 'tag' or not text:
 517             yield text
 518             continue
 519
 520         # skip literal text (math, literal, raw, ...)
 521         if ttype == 'literal':
 522             prev_token_last_char = text[-1:]
 523             yield text
 524             continue
 525
 526         last_char = text[-1:] # Remember last char before processing.
 527
 528         text = processEscapes(text)
 529
 530         if convert_quot:
 531             text = re.sub('&quot;', '"', text)
 532
 533         if do_dashes == 1:
 534             text = educateDashes(text)
 535         elif do_dashes == 2:
 536             text = educateDashesOldSchool(text)
 537         elif do_dashes == 3:
 538             text = educateDashesOldSchoolInverted(text)
 539
 540         if do_ellipses:
 541             text = educateEllipses(text)
 542
 543         # Note: backticks need to be processed before quotes.
 544         if do_backticks:
 545             text = educateBackticks(text, language)
 546
 547         if do_backticks == 2:
 548             text = educateSingleBackticks(text, language)
 549
 550         if do_quotes:
 551             text = educateQuotes(prev_token_last_char+text, language)[1:]
 552
 553         if do_stupefy:
 554             text = stupefyEntities(text, language)
 555
 556         # Remember last char as context for the next token
 557         prev_token_last_char = last_char
 558
 559         text = processEscapes(text, restore=True)
 560
 561         yield text
 562
 563
 564
 565 def educateQuotes(text, language='en'):
 566     """
 567     Parameter:  - text string (unicode or bytes).
 568                 - language (`BCP 47` language tag.)
 569     Returns:    The `text`, with "educated" curly quote characters.
 570
 571     Example input:  "Isn't this fun?"
 572     Example output: “Isn’t this fun?“;
 573     """
 574
 575     smart = smartchars(language)
 576
 577     # oldtext = text
 578     punct_class = r"""[!"#\$\%'()*+,-.\/:;<=>?\@\[\\\]\^_`{|}~]"""
 579
 580     # Special case if the very first character is a quote
 581     # followed by punctuation at a non-word-break.
 582     # Close the quotes by brute force:
 583     text = re.sub(r"""^'(?=%s\\B)""" % (punct_class,), smart.csquote, text)
 584     text = re.sub(r"""^"(?=%s\\B)""" % (punct_class,), smart.cpquote, text)
 585
 586     # Special case for double sets of quotes, e.g.:
 587     #   <p>He said, "'Quoted' words in a larger quote."</p>
 588     text = re.sub(r""""'(?=\w)""", smart.opquote+smart.osquote, text)
 589     text = re.sub(r"""'"(?=\w)""", smart.osquote+smart.opquote, text)
 590
 591     # Special case for decade abbreviations (the '80s):
 592     text = re.sub(r"""\b'(?=\d{2}s)""", smart.csquote, text)
 593
 594     close_class = r"""[^\ \t\r\n\[\{\(\-]"""
 595     dec_dashes = r"""&#8211;|&#8212;"""
 596
 597     # Get most opening single quotes:
 598     opening_single_quotes_regex = re.compile(r"""
 599                     (
 600                             \s          |   # a whitespace char, or
 601                             &nbsp;      |   # a non-breaking space entity, or
 602                             --          |   # dashes, or
 603                             &[mn]dash;  |   # named dash entities
 604                             %s          |   # or decimal entities
 605                             &\#x201[34];    # or hex
 606                     )
 607                     '                 # the quote
 608                     (?=\w)            # followed by a word character
 609                     """ % (dec_dashes,), re.VERBOSE)
 610     text = opening_single_quotes_regex.sub(r'\1'+smart.osquote, text)
 611
 612     closing_single_quotes_regex = re.compile(r"""
 613                     (%s)
 614                     '
 615                     (?!\s | s\b | \d)
 616                     """ % (close_class,), re.VERBOSE)
 617     text = closing_single_quotes_regex.sub(r'\1'+smart.csquote, text)
 618
 619     closing_single_quotes_regex = re.compile(r"""
 620                     (%s)
 621                     '
 622                     (\s | s\b)
 623                     """ % (close_class,), re.VERBOSE)
 624     text = closing_single_quotes_regex.sub(r'\1%s\2' % smart.csquote, text)
 625
 626     # Any remaining single quotes should be opening ones:
 627     text = re.sub(r"""'""", smart.osquote, text)
 628
 629     # Get most opening double quotes:
 630     opening_double_quotes_regex = re.compile(r"""
 631                     (
 632                             \s          |   # a whitespace char, or
 633                             &nbsp;      |   # a non-breaking space entity, or
 634                             --          |   # dashes, or
 635                             &[mn]dash;  |   # named dash entities
 636                             %s          |   # or decimal entities
 637                             &\#x201[34];    # or hex
 638                     )
 639                     "                 # the quote
 640                     (?=\w)            # followed by a word character
 641                     """ % (dec_dashes,), re.VERBOSE)
 642     text = opening_double_quotes_regex.sub(r'\1'+smart.opquote, text)
 643
 644     # Double closing quotes:
 645     closing_double_quotes_regex = re.compile(r"""
 646                     #(%s)?   # character that indicates the quote should be closing
 647                     "
 648                     (?=\s)
 649                     """ % (close_class,), re.VERBOSE)
 650     text = closing_double_quotes_regex.sub(smart.cpquote, text)
 651
 652     closing_double_quotes_regex = re.compile(r"""
 653                     (%s)   # character that indicates the quote should be closing
 654                     "
 655                     """ % (close_class,), re.VERBOSE)
 656     text = closing_double_quotes_regex.sub(r'\1'+smart.cpquote, text)
 657
 658     # Any remaining quotes should be opening ones.
 659     text = re.sub(r'"', smart.opquote, text)
 660
 661     return text
 662
 663
 664 def educateBackticks(text, language='en'):
 665     """
 666     Parameter:  String (unicode or bytes).
 667     Returns:    The `text`, with ``backticks'' -style double quotes
 668                 translated into HTML curly quote entities.
 669     Example input:  ``Isn't this fun?''
 670     Example output: “Isn't this fun?“;
 671     """
 672     smart = smartchars(language)
 673
 674     text = re.sub(r"""``""", smart.opquote, text)
 675     text = re.sub(r"""''""", smart.cpquote, text)
 676     return text
 677
 678
 679 def educateSingleBackticks(text, language='en'):
 680     """
 681     Parameter:  String (unicode or bytes).
 682     Returns:    The `text`, with `backticks' -style single quotes
 683                 translated into HTML curly quote entities.
 684
 685     Example input:  `Isn't this fun?'
 686     Example output: ‘Isn’t this fun?’
 687     """
 688     smart = smartchars(language)
 689
 690     text = re.sub(r"""`""", smart.osquote, text)
 691     text = re.sub(r"""'""", smart.csquote, text)
 692     return text
 693
 694
 695 def educateDashes(text):
 696     """
 697     Parameter:  String (unicode or bytes).
 698     Returns:    The `text`, with each instance of "--" translated to
 699                 an em-dash character.
 700     """
 701
 702     text = re.sub(r"""---""", smartchars.endash, text) # en  (yes, backwards)
 703     text = re.sub(r"""--""", smartchars.emdash, text) # em (yes, backwards)
 704     return text
 705
 706
 707 def educateDashesOldSchool(text):
 708     """
 709     Parameter:  String (unicode or bytes).
 710     Returns:    The `text`, with each instance of "--" translated to
 711                 an en-dash character, and each "---" translated to
 712                 an em-dash character.
 713     """
 714
 715     text = re.sub(r"""---""", smartchars.emdash, text)
 716     text = re.sub(r"""--""", smartchars.endash, text)
 717     return text
 718
 719
 720 def educateDashesOldSchoolInverted(text):
 721     """
 722     Parameter:  String (unicode or bytes).
 723     Returns:    The `text`, with each instance of "--" translated to
 724                 an em-dash character, and each "---" translated to
 725                 an en-dash character. Two reasons why: First, unlike the
 726                 en- and em-dash syntax supported by
 727                 EducateDashesOldSchool(), it's compatible with existing
 728                 entries written before SmartyPants 1.1, back when "--" was
 729                 only used for em-dashes.  Second, em-dashes are more
 730                 common than en-dashes, and so it sort of makes sense that
 731                 the shortcut should be shorter to type. (Thanks to Aaron
 732                 Swartz for the idea.)
 733     """
 734     text = re.sub(r"""---""", smartchars.endash, text)    # em
 735     text = re.sub(r"""--""", smartchars.emdash, text)    # en
 736     return text
 737
 738
 739
 740 def educateEllipses(text):
 741     """
 742     Parameter:  String (unicode or bytes).
 743     Returns:    The `text`, with each instance of "..." translated to
 744                 an ellipsis character.
 745
 746     Example input:  Huh...?
 747     Example output: Huh&#8230;?
 748     """
 749
 750     text = re.sub(r"""\.\.\.""", smartchars.ellipsis, text)
 751     text = re.sub(r"""\. \. \.""", smartchars.ellipsis, text)
 752     return text
 753
 754
 755 def stupefyEntities(text, language='en'):
 756     """
 757     Parameter:  String (unicode or bytes).
 758     Returns:    The `text`, with each SmartyPants character translated to
 759                 its ASCII counterpart.
 760
 761     Example input:  “Hello — world.”
 762     Example output: "Hello -- world."
 763     """
 764     smart = smartchars(language)
 765
 766     text = re.sub(smart.endash, "-", text)  # en-dash
 767     text = re.sub(smart.emdash, "--", text) # em-dash
 768
 769     text = re.sub(smart.osquote, "'", text)  # open single quote
 770     text = re.sub(smart.csquote, "'", text)  # close single quote
 771
 772     text = re.sub(smart.opquote, '"', text)  # open double quote
 773     text = re.sub(smart.cpquote, '"', text)  # close double quote
 774
 775     text = re.sub(smart.ellipsis, '...', text)# ellipsis
 776
 777     return text
 778
 779
 780 def processEscapes(text, restore=False):
 781     r"""
 782     Parameter:  String (unicode or bytes).
 783     Returns:    The `text`, with after processing the following backslash
 784                 escape sequences. This is useful if you want to force a "dumb"
 785                 quote or other character to appear.
 786
 787                 Escape  Value
 788                 ------  -----
 789                 \\      &#92;
 790                 \"      &#34;
 791                 \'      &#39;
 792                 \.      &#46;
 793                 \-      &#45;
 794                 \`      &#96;
 795     """
 796     replacements = ((r'\\', r'&#92;'),
 797                     (r'\"', r'&#34;'),
 798                     (r"\'", r'&#39;'),
 799                     (r'\.', r'&#46;'),
 800                     (r'\-', r'&#45;'),
 801                     (r'\`', r'&#96;'))
 802     if restore:
 803         for (ch, rep) in replacements:
 804             text = text.replace(rep, ch[1])
 805     else:
 806         for (ch, rep) in replacements:
 807             text = text.replace(ch, rep)
 808
 809     return text
 810
 811
 812 def tokenize(text):
 813     """
 814     Parameter:  String containing HTML markup.
 815     Returns:    An iterator that yields the tokens comprising the input
 816                 string. Each token is either a tag (possibly with nested,
 817                 tags contained therein, such as <a href="<MTFoo>">, or a
 818                 run of text between tags. Each yielded element is a
 819                 two-element tuple; the first is either 'tag' or 'text';
 820                 the second is the actual value.
 821
 822     Based on the _tokenize() subroutine from Brad Choate's MTRegex plugin.
 823         <http://www.bradchoate.com/past/mtregex.php>
 824     """
 825
 826     pos = 0
 827     length = len(text)
 828     # tokens = []
 829
 830     depth = 6
 831     nested_tags = "|".join(['(?:<(?:[^<>]',] * depth) + (')*>)' * depth)
 832     #match = r"""(?: <! ( -- .*? -- \s* )+ > ) |  # comments
 833     #               (?: <\? .*? \?> ) |  # directives
 834     #               %s  # nested tags       """ % (nested_tags,)
 835     tag_soup = re.compile(r"""([^<]*)(<[^>]*>)""")
 836
 837     token_match = tag_soup.search(text)
 838
 839     previous_end = 0
 840     while token_match is not None:
 841         if token_match.group(1):
 842             yield ('text', token_match.group(1))
 843
 844         yield ('tag', token_match.group(2))
 845
 846         previous_end = token_match.end()
 847         token_match = tag_soup.search(text, token_match.end())
 848
 849     if previous_end < len(text):
 850         yield ('text', text[previous_end:])
 851
 852
 853
 854 if __name__ == "__main__":
 855
 856     import locale
 857
 858     try:
 859         locale.setlocale(locale.LC_ALL, '')
 860     except:
 861         pass
 862
 863     from docutils.core import publish_string
 864     docstring_html = publish_string(__doc__, writer_name='html')
 865
 866     print docstring_html
 867
 868
 869     # Unit test output goes out stderr.
 870     import unittest
 871     sp = smartyPants
 872
 873     class TestSmartypantsAllAttributes(unittest.TestCase):
 874         # the default attribute is "1", which means "all".
 875
 876         def test_dates(self):
 877             self.assertEqual(sp("1440-80's"), u"1440-80’s")
 878             self.assertEqual(sp("1440-'80s"), u"1440-‘80s")
 879             self.assertEqual(sp("1440---'80s"), u"1440–‘80s")
 880             self.assertEqual(sp("1960s"), "1960s")  # no effect.
 881             self.assertEqual(sp("1960's"), u"1960’s")
 882             self.assertEqual(sp("one two '60s"), u"one two ‘60s")
 883             self.assertEqual(sp("'60s"), u"‘60s")
 884
 885         def test_ordinal_numbers(self):
 886             self.assertEqual(sp("21st century"), "21st century")  # no effect.
 887             self.assertEqual(sp("3rd"), "3rd")  # no effect.
 888
 889         def test_educated_quotes(self):
 890             self.assertEqual(sp('''"Isn't this fun?"'''), u'“Isn’t this fun?”')
 891
 892         def test_html_tags(self):
 893             text = '<a src="foo">more</a>'
 894             self.assertEqual(sp(text), text)
 895
 896     unittest.main()
 897
 898
 899
 900
 901 __author__ = "Chad Miller <smartypantspy@chad.org>"
 902 __version__ = "1.5_1.6: Fri, 27 Jul 2007 07:06:40 -0400"
 903 __url__ = "http://wiki.chad.org/SmartyPantsPy"
 904 __description__ = "Smart-quotes, smart-ellipses, and smart-dashes for weblog entries in pyblosxom"