Lib/test/test_codeccallbacks.py

   1 import test.test_support, unittest
   2 import sys, codecs, htmlentitydefs, unicodedata
   3
   4 class PosReturn:
   5     # this can be used for configurable callbacks
   6
   7     def __init__(self):
   8         self.pos = 0
   9
  10     def handle(self, exc):
  11         oldpos = self.pos
  12         realpos = oldpos
  13         if realpos<0:
  14             realpos = len(exc.object) + realpos
  15         # if we don't advance this time, terminate on the next call
  16         # otherwise we'd get an endless loop
  17         if realpos <= exc.start:
  18             self.pos = len(exc.object)
  19         return (u"<?>", oldpos)
  20
  21 # A UnicodeEncodeError object with a bad start attribute
  22 class BadStartUnicodeEncodeError(UnicodeEncodeError):
  23     def __init__(self):
  24         UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
  25         self.start = []
  26
  27 # A UnicodeEncodeError object with a bad object attribute
  28 class BadObjectUnicodeEncodeError(UnicodeEncodeError):
  29     def __init__(self):
  30         UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
  31         self.object = []
  32
  33 # A UnicodeDecodeError object without an end attribute
  34 class NoEndUnicodeDecodeError(UnicodeDecodeError):
  35     def __init__(self):
  36         UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
  37         del self.end
  38
  39 # A UnicodeDecodeError object with a bad object attribute
  40 class BadObjectUnicodeDecodeError(UnicodeDecodeError):
  41     def __init__(self):
  42         UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
  43         self.object = []
  44
  45 # A UnicodeTranslateError object without a start attribute
  46 class NoStartUnicodeTranslateError(UnicodeTranslateError):
  47     def __init__(self):
  48         UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
  49         del self.start
  50
  51 # A UnicodeTranslateError object without an end attribute
  52 class NoEndUnicodeTranslateError(UnicodeTranslateError):
  53     def __init__(self):
  54         UnicodeTranslateError.__init__(self,  u"", 0, 1, "bad")
  55         del self.end
  56
  57 # A UnicodeTranslateError object without an object attribute
  58 class NoObjectUnicodeTranslateError(UnicodeTranslateError):
  59     def __init__(self):
  60         UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
  61         del self.object
  62
  63 class CodecCallbackTest(unittest.TestCase):
  64
  65     def test_xmlcharrefreplace(self):
  66         # replace unencodable characters which numeric character entities.
  67         # For ascii, latin-1 and charmaps this is completely implemented
  68         # in C and should be reasonably fast.
  69         s = u"\u30b9\u30d1\u30e2 \xe4nd eggs"
  70         self.assertEqual(
  71             s.encode("ascii", "xmlcharrefreplace"),
  72             "&#12473;&#12497;&#12514; &#228;nd eggs"
  73         )
  74         self.assertEqual(
  75             s.encode("latin-1", "xmlcharrefreplace"),
  76             "&#12473;&#12497;&#12514; \xe4nd eggs"
  77         )
  78
  79     def test_xmlcharnamereplace(self):
  80         # This time use a named character entity for unencodable
  81         # characters, if one is available.
  82
  83         def xmlcharnamereplace(exc):
  84             if not isinstance(exc, UnicodeEncodeError):
  85                 raise TypeError("don't know how to handle %r" % exc)
  86             l = []
  87             for c in exc.object[exc.start:exc.end]:
  88                 try:
  89                     l.append(u"&%s;" % htmlentitydefs.codepoint2name[ord(c)])
  90                 except KeyError:
  91                     l.append(u"&#%d;" % ord(c))
  92             return (u"".join(l), exc.end)
  93
  94         codecs.register_error(
  95             "test.xmlcharnamereplace", xmlcharnamereplace)
  96
  97         sin = u"\xab\u211c\xbb = \u2329\u1234\u20ac\u232a"
  98         sout = "&laquo;&real;&raquo; = &lang;&#4660;&euro;&rang;"
  99         self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout)
 100         sout = "\xab&real;\xbb = &lang;&#4660;&euro;&rang;"
 101         self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout)
 102         sout = "\xab&real;\xbb = &lang;&#4660;\xa4&rang;"
 103         self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout)
 104
 105     def test_uninamereplace(self):
 106         # We're using the names from the unicode database this time,
 107         # and we're doing "syntax highlighting" here, i.e. we include
 108         # the replaced text in ANSI escape sequences. For this it is
 109         # useful that the error handler is not called for every single
 110         # unencodable character, but for a complete sequence of
 111         # unencodable characters, otherwise we would output many
 112         # unnecessary escape sequences.
 113
 114         def uninamereplace(exc):
 115             if not isinstance(exc, UnicodeEncodeError):
 116                 raise TypeError("don't know how to handle %r" % exc)
 117             l = []
 118             for c in exc.object[exc.start:exc.end]:
 119                 l.append(unicodedata.name(c, u"0x%x" % ord(c)))
 120             return (u"\033[1m%s\033[0m" % u", ".join(l), exc.end)
 121
 122         codecs.register_error(
 123             "test.uninamereplace", uninamereplace)
 124
 125         sin = u"\xac\u1234\u20ac\u8000"
 126         sout = "\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
 127         self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout)
 128
 129         sout = "\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
 130         self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout)
 131
 132         sout = "\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m"
 133         self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout)
 134
 135     def test_backslashescape(self):
 136         # Does the same as the "unicode-escape" encoding, but with different
 137         # base encodings.
 138         sin = u"a\xac\u1234\u20ac\u8000"
 139         if sys.maxunicode > 0xffff:
 140             sin += unichr(sys.maxunicode)
 141         sout = "a\\xac\\u1234\\u20ac\\u8000"
 142         if sys.maxunicode > 0xffff:
 143             sout += "\\U%08x" % sys.maxunicode
 144         self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
 145
 146         sout = "a\xac\\u1234\\u20ac\\u8000"
 147         if sys.maxunicode > 0xffff:
 148             sout += "\\U%08x" % sys.maxunicode
 149         self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
 150
 151         sout = "a\xac\\u1234\xa4\\u8000"
 152         if sys.maxunicode > 0xffff:
 153             sout += "\\U%08x" % sys.maxunicode
 154         self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
 155
 156     def test_decoding_callbacks(self):
 157         # This is a test for a decoding callback handler
 158         # that allows the decoding of the invalid sequence
 159         # "\xc0\x80" and returns "\x00" instead of raising an error.
 160         # All other illegal sequences will be handled strictly.
 161         def relaxedutf8(exc):
 162             if not isinstance(exc, UnicodeDecodeError):
 163                 raise TypeError("don't know how to handle %r" % exc)
 164             if exc.object[exc.start:exc.start+2] == "\xc0\x80":
 165                 return (u"\x00", exc.start+2) # retry after two bytes
 166             else:
 167                 raise exc
 168
 169         codecs.register_error("test.relaxedutf8", relaxedutf8)
 170
 171         # all the "\xc0\x80" will be decoded to "\x00"
 172         sin = "a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80"
 173         sout = u"a\x00b\x00c\xfc\x00\x00"
 174         self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout)
 175
 176         # "\xc0\x81" is not valid and a UnicodeDecodeError will be raised
 177         sin = "\xc0\x80\xc0\x81"
 178         self.assertRaises(UnicodeDecodeError, sin.decode,
 179                           "utf-8", "test.relaxedutf8")
 180
 181     def test_charmapencode(self):
 182         # For charmap encodings the replacement string will be
 183         # mapped through the encoding again. This means, that
 184         # to be able to use e.g. the "replace" handler, the
 185         # charmap has to have a mapping for "?".
 186         charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
 187         sin = u"abc"
 188         sout = "AABBCC"
 189         self.assertEquals(codecs.charmap_encode(sin, "strict", charmap)[0], sout)
 190
 191         sin = u"abcA"
 192         self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap)
 193
 194         charmap[ord("?")] = "XYZ"
 195         sin = u"abcDEF"
 196         sout = "AABBCCXYZXYZXYZ"
 197         self.assertEquals(codecs.charmap_encode(sin, "replace", charmap)[0], sout)
 198
 199         charmap[ord("?")] = u"XYZ"
 200         self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
 201
 202         charmap[ord("?")] = u"XYZ"
 203         self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
 204
 205     def test_decodeunicodeinternal(self):
 206         self.assertRaises(
 207             UnicodeDecodeError,
 208             "\x00\x00\x00\x00\x00".decode,
 209             "unicode-internal",
 210         )
 211         if sys.maxunicode > 0xffff:
 212             def handler_unicodeinternal(exc):
 213                 if not isinstance(exc, UnicodeDecodeError):
 214                     raise TypeError("don't know how to handle %r" % exc)
 215                 return (u"\x01", 1)
 216
 217             self.assertEqual(
 218                 "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
 219                 u"\u0000"
 220             )
 221
 222             self.assertEqual(
 223                 "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
 224                 u"\u0000\ufffd"
 225             )
 226
 227             codecs.register_error("test.hui", handler_unicodeinternal)
 228
 229             self.assertEqual(
 230                 "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
 231                 u"\u0000\u0001\u0000"
 232             )
 233
 234     def test_callbacks(self):
 235         def handler1(exc):
 236             if not isinstance(exc, UnicodeEncodeError) \
 237                and not isinstance(exc, UnicodeDecodeError):
 238                 raise TypeError("don't know how to handle %r" % exc)
 239             l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
 240             return (u"[%s]" % u"".join(l), exc.end)
 241
 242         codecs.register_error("test.handler1", handler1)
 243
 244         def handler2(exc):
 245             if not isinstance(exc, UnicodeDecodeError):
 246                 raise TypeError("don't know how to handle %r" % exc)
 247             l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
 248             return (u"[%s]" % u"".join(l), exc.end+1) # skip one character
 249
 250         codecs.register_error("test.handler2", handler2)
 251
 252         s = "\x00\x81\x7f\x80\xff"
 253
 254         self.assertEqual(
 255             s.decode("ascii", "test.handler1"),
 256             u"\x00[<129>]\x7f[<128>][<255>]"
 257         )
 258         self.assertEqual(
 259             s.decode("ascii", "test.handler2"),
 260             u"\x00[<129>][<128>]"
 261         )
 262
 263         self.assertEqual(
 264             "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
 265             u"\u3042[<92><117><51><120>]xx"
 266         )
 267
 268         self.assertEqual(
 269             "\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
 270             u"\u3042[<92><117><51><120><120>]"
 271         )
 272
 273         self.assertEqual(
 274             codecs.charmap_decode("abc", "test.handler1", {ord("a"): u"z"})[0],
 275             u"z[<98>][<99>]"
 276         )
 277
 278         self.assertEqual(
 279             u"g\xfc\xdfrk".encode("ascii", "test.handler1"),
 280             u"g[<252><223>]rk"
 281         )
 282
 283         self.assertEqual(
 284             u"g\xfc\xdf".encode("ascii", "test.handler1"),
 285             u"g[<252><223>]"
 286         )
 287
 288     def test_longstrings(self):
 289         # test long strings to check for memory overflow problems
 290         errors = [ "strict", "ignore", "replace", "xmlcharrefreplace",
 291                    "backslashreplace"]
 292         # register the handlers under different names,
 293         # to prevent the codec from recognizing the name
 294         for err in errors:
 295             codecs.register_error("test." + err, codecs.lookup_error(err))
 296         l = 1000
 297         errors += [ "test." + err for err in errors ]
 298         for uni in [ s*l for s in (u"x", u"\u3042", u"a\xe4") ]:
 299             for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15",
 300                         "utf-8", "utf-7", "utf-16", "utf-32"):
 301                 for err in errors:
 302                     try:
 303                         uni.encode(enc, err)
 304                     except UnicodeError:
 305                         pass
 306
 307     def check_exceptionobjectargs(self, exctype, args, msg):
 308         # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion
 309         # check with one missing argument
 310         self.assertRaises(TypeError, exctype, *args[:-1])
 311         # check with one argument too much
 312         self.assertRaises(TypeError, exctype, *(args + ["too much"]))
 313         # check with one argument of the wrong type
 314         wrongargs = [ "spam", u"eggs", 42, 1.0, None ]
 315         for i in xrange(len(args)):
 316             for wrongarg in wrongargs:
 317                 if type(wrongarg) is type(args[i]):
 318                     continue
 319                 # build argument array
 320                 callargs = []
 321                 for j in xrange(len(args)):
 322                     if i==j:
 323                         callargs.append(wrongarg)
 324                     else:
 325                         callargs.append(args[i])
 326                 self.assertRaises(TypeError, exctype, *callargs)
 327
 328         # check with the correct number and type of arguments
 329         exc = exctype(*args)
 330         self.assertEquals(str(exc), msg)
 331
 332     def test_unicodeencodeerror(self):
 333         self.check_exceptionobjectargs(
 334             UnicodeEncodeError,
 335             ["ascii", u"g\xfcrk", 1, 2, "ouch"],
 336             "'ascii' codec can't encode character u'\\xfc' in position 1: ouch"
 337         )
 338         self.check_exceptionobjectargs(
 339             UnicodeEncodeError,
 340             ["ascii", u"g\xfcrk", 1, 4, "ouch"],
 341             "'ascii' codec can't encode characters in position 1-3: ouch"
 342         )
 343         self.check_exceptionobjectargs(
 344             UnicodeEncodeError,
 345             ["ascii", u"\xfcx", 0, 1, "ouch"],
 346             "'ascii' codec can't encode character u'\\xfc' in position 0: ouch"
 347         )
 348         self.check_exceptionobjectargs(
 349             UnicodeEncodeError,
 350             ["ascii", u"\u0100x", 0, 1, "ouch"],
 351             "'ascii' codec can't encode character u'\\u0100' in position 0: ouch"
 352         )
 353         self.check_exceptionobjectargs(
 354             UnicodeEncodeError,
 355             ["ascii", u"\uffffx", 0, 1, "ouch"],
 356             "'ascii' codec can't encode character u'\\uffff' in position 0: ouch"
 357         )
 358         if sys.maxunicode > 0xffff:
 359             self.check_exceptionobjectargs(
 360                 UnicodeEncodeError,
 361                 ["ascii", u"\U00010000x", 0, 1, "ouch"],
 362                 "'ascii' codec can't encode character u'\\U00010000' in position 0: ouch"
 363             )
 364
 365     def test_unicodedecodeerror(self):
 366         self.check_exceptionobjectargs(
 367             UnicodeDecodeError,
 368             ["ascii", "g\xfcrk", 1, 2, "ouch"],
 369             "'ascii' codec can't decode byte 0xfc in position 1: ouch"
 370         )
 371         self.check_exceptionobjectargs(
 372             UnicodeDecodeError,
 373             ["ascii", "g\xfcrk", 1, 3, "ouch"],
 374             "'ascii' codec can't decode bytes in position 1-2: ouch"
 375         )
 376
 377     def test_unicodetranslateerror(self):
 378         self.check_exceptionobjectargs(
 379             UnicodeTranslateError,
 380             [u"g\xfcrk", 1, 2, "ouch"],
 381             "can't translate character u'\\xfc' in position 1: ouch"
 382         )
 383         self.check_exceptionobjectargs(
 384             UnicodeTranslateError,
 385             [u"g\u0100rk", 1, 2, "ouch"],
 386             "can't translate character u'\\u0100' in position 1: ouch"
 387         )
 388         self.check_exceptionobjectargs(
 389             UnicodeTranslateError,
 390             [u"g\uffffrk", 1, 2, "ouch"],
 391             "can't translate character u'\\uffff' in position 1: ouch"
 392         )
 393         if sys.maxunicode > 0xffff:
 394             self.check_exceptionobjectargs(
 395                 UnicodeTranslateError,
 396                 [u"g\U00010000rk", 1, 2, "ouch"],
 397                 "can't translate character u'\\U00010000' in position 1: ouch"
 398             )
 399         self.check_exceptionobjectargs(
 400             UnicodeTranslateError,
 401             [u"g\xfcrk", 1, 3, "ouch"],
 402             "can't translate characters in position 1-2: ouch"
 403         )
 404
 405     def test_badandgoodstrictexceptions(self):
 406         # "strict" complains about a non-exception passed in
 407         self.assertRaises(
 408             TypeError,
 409             codecs.strict_errors,
 410             42
 411         )
 412         # "strict" complains about the wrong exception type
 413         self.assertRaises(
 414             Exception,
 415             codecs.strict_errors,
 416             Exception("ouch")
 417         )
 418
 419         # If the correct exception is passed in, "strict" raises it
 420         self.assertRaises(
 421             UnicodeEncodeError,
 422             codecs.strict_errors,
 423             UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")
 424         )
 425
 426     def test_badandgoodignoreexceptions(self):
 427         # "ignore" complains about a non-exception passed in
 428         self.assertRaises(
 429            TypeError,
 430            codecs.ignore_errors,
 431            42
 432         )
 433         # "ignore" complains about the wrong exception type
 434         self.assertRaises(
 435            TypeError,
 436            codecs.ignore_errors,
 437            UnicodeError("ouch")
 438         )
 439         # If the correct exception is passed in, "ignore" returns an empty replacement
 440         self.assertEquals(
 441             codecs.ignore_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
 442             (u"", 1)
 443         )
 444         self.assertEquals(
 445             codecs.ignore_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
 446             (u"", 1)
 447         )
 448         self.assertEquals(
 449             codecs.ignore_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
 450             (u"", 1)
 451         )
 452
 453     def test_badandgoodreplaceexceptions(self):
 454         # "replace" complains about a non-exception passed in
 455         self.assertRaises(
 456            TypeError,
 457            codecs.replace_errors,
 458            42
 459         )
 460         # "replace" complains about the wrong exception type
 461         self.assertRaises(
 462            TypeError,
 463            codecs.replace_errors,
 464            UnicodeError("ouch")
 465         )
 466         self.assertRaises(
 467             TypeError,
 468             codecs.replace_errors,
 469             BadObjectUnicodeEncodeError()
 470         )
 471         self.assertRaises(
 472             TypeError,
 473             codecs.replace_errors,
 474             BadObjectUnicodeDecodeError()
 475         )
 476         # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement
 477         self.assertEquals(
 478             codecs.replace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
 479             (u"?", 1)
 480         )
 481         self.assertEquals(
 482             codecs.replace_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
 483             (u"\ufffd", 1)
 484         )
 485         self.assertEquals(
 486             codecs.replace_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
 487             (u"\ufffd", 1)
 488         )
 489
 490     def test_badandgoodxmlcharrefreplaceexceptions(self):
 491         # "xmlcharrefreplace" complains about a non-exception passed in
 492         self.assertRaises(
 493            TypeError,
 494            codecs.xmlcharrefreplace_errors,
 495            42
 496         )
 497         # "xmlcharrefreplace" complains about the wrong exception types
 498         self.assertRaises(
 499            TypeError,
 500            codecs.xmlcharrefreplace_errors,
 501            UnicodeError("ouch")
 502         )
 503         # "xmlcharrefreplace" can only be used for encoding
 504         self.assertRaises(
 505             TypeError,
 506             codecs.xmlcharrefreplace_errors,
 507             UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
 508         )
 509         self.assertRaises(
 510             TypeError,
 511             codecs.xmlcharrefreplace_errors,
 512             UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
 513         )
 514         # Use the correct exception
 515         cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042)
 516         s = "".join(unichr(c) for c in cs)
 517         self.assertEquals(
 518             codecs.xmlcharrefreplace_errors(
 519                 UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
 520             ),
 521             (u"".join(u"&#%d;" % ord(c) for c in s), len(s))
 522         )
 523
 524     def test_badandgoodbackslashreplaceexceptions(self):
 525         # "backslashreplace" complains about a non-exception passed in
 526         self.assertRaises(
 527            TypeError,
 528            codecs.backslashreplace_errors,
 529            42
 530         )
 531         # "backslashreplace" complains about the wrong exception types
 532         self.assertRaises(
 533            TypeError,
 534            codecs.backslashreplace_errors,
 535            UnicodeError("ouch")
 536         )
 537         # "backslashreplace" can only be used for encoding
 538         self.assertRaises(
 539             TypeError,
 540             codecs.backslashreplace_errors,
 541             UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
 542         )
 543         self.assertRaises(
 544             TypeError,
 545             codecs.backslashreplace_errors,
 546             UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
 547         )
 548         # Use the correct exception
 549         self.assertEquals(
 550             codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
 551             (u"\\u3042", 1)
 552         )
 553         self.assertEquals(
 554             codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\x00", 0, 1, "ouch")),
 555             (u"\\x00", 1)
 556         )
 557         self.assertEquals(
 558             codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\xff", 0, 1, "ouch")),
 559             (u"\\xff", 1)
 560         )
 561         self.assertEquals(
 562             codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u0100", 0, 1, "ouch")),
 563             (u"\\u0100", 1)
 564         )
 565         self.assertEquals(
 566             codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\uffff", 0, 1, "ouch")),
 567             (u"\\uffff", 1)
 568         )
 569         if sys.maxunicode>0xffff:
 570             self.assertEquals(
 571                 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U00010000", 0, 1, "ouch")),
 572                 (u"\\U00010000", 1)
 573             )
 574             self.assertEquals(
 575                 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U0010ffff", 0, 1, "ouch")),
 576                 (u"\\U0010ffff", 1)
 577             )
 578
 579     def test_badhandlerresults(self):
 580         results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
 581         encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15")
 582
 583         for res in results:
 584             codecs.register_error("test.badhandler", lambda x: res)
 585             for enc in encs:
 586                 self.assertRaises(
 587                     TypeError,
 588                     u"\u3042".encode,
 589                     enc,
 590                     "test.badhandler"
 591                 )
 592             for (enc, bytes) in (
 593                 ("ascii", "\xff"),
 594                 ("utf-8", "\xff"),
 595                 ("utf-7", "+x-"),
 596                 ("unicode-internal", "\x00"),
 597             ):
 598                 self.assertRaises(
 599                     TypeError,
 600                     bytes.decode,
 601                     enc,
 602                     "test.badhandler"
 603                 )
 604
 605     def test_lookup(self):
 606         self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
 607         self.assertEquals(codecs.ignore_errors, codecs.lookup_error("ignore"))
 608         self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
 609         self.assertEquals(
 610             codecs.xmlcharrefreplace_errors,
 611             codecs.lookup_error("xmlcharrefreplace")
 612         )
 613         self.assertEquals(
 614             codecs.backslashreplace_errors,
 615             codecs.lookup_error("backslashreplace")
 616         )
 617
 618     def test_unencodablereplacement(self):
 619         def unencrepl(exc):
 620             if isinstance(exc, UnicodeEncodeError):
 621                 return (u"\u4242", exc.end)
 622             else:
 623                 raise TypeError("don't know how to handle %r" % exc)
 624         codecs.register_error("test.unencreplhandler", unencrepl)
 625         for enc in ("ascii", "iso-8859-1", "iso-8859-15"):
 626             self.assertRaises(
 627                 UnicodeEncodeError,
 628                 u"\u4242".encode,
 629                 enc,
 630                 "test.unencreplhandler"
 631             )
 632
 633     def test_badregistercall(self):
 634         # enhance coverage of:
 635         # Modules/_codecsmodule.c::register_error()
 636         # Python/codecs.c::PyCodec_RegisterError()
 637         self.assertRaises(TypeError, codecs.register_error, 42)
 638         self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42)
 639
 640     def test_badlookupcall(self):
 641         # enhance coverage of:
 642         # Modules/_codecsmodule.c::lookup_error()
 643         self.assertRaises(TypeError, codecs.lookup_error)
 644
 645     def test_unknownhandler(self):
 646         # enhance coverage of:
 647         # Modules/_codecsmodule.c::lookup_error()
 648         self.assertRaises(LookupError, codecs.lookup_error, "test.unknown")
 649
 650     def test_xmlcharrefvalues(self):
 651         # enhance coverage of:
 652         # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
 653         # and inline implementations
 654         v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000)
 655         if sys.maxunicode>=100000:
 656             v += (100000, 500000, 1000000)
 657         s = u"".join([unichr(x) for x in v])
 658         codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
 659         for enc in ("ascii", "iso-8859-15"):
 660             for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"):
 661                 s.encode(enc, err)
 662
 663     def test_decodehelper(self):
 664         # enhance coverage of:
 665         # Objects/unicodeobject.c::unicode_decode_call_errorhandler()
 666         # and callers
 667         self.assertRaises(LookupError, "\xff".decode, "ascii", "test.unknown")
 668
 669         def baddecodereturn1(exc):
 670             return 42
 671         codecs.register_error("test.baddecodereturn1", baddecodereturn1)
 672         self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1")
 673         self.assertRaises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1")
 674         self.assertRaises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1")
 675         self.assertRaises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
 676         self.assertRaises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
 677         self.assertRaises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
 678
 679         def baddecodereturn2(exc):
 680             return (u"?", None)
 681         codecs.register_error("test.baddecodereturn2", baddecodereturn2)
 682         self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2")
 683
 684         handler = PosReturn()
 685         codecs.register_error("test.posreturn", handler.handle)
 686
 687         # Valid negative position
 688         handler.pos = -1
 689         self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
 690
 691         # Valid negative position
 692         handler.pos = -2
 693         self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?><?>")
 694
 695         # Negative position out of bounds
 696         handler.pos = -3
 697         self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
 698
 699         # Valid positive position
 700         handler.pos = 1
 701         self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
 702
 703         # Largest valid positive position (one beyond end of input)
 704         handler.pos = 2
 705         self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>")
 706
 707         # Invalid positive position
 708         handler.pos = 3
 709         self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
 710
 711         # Restart at the "0"
 712         handler.pos = 6
 713         self.assertEquals("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"<?>0")
 714
 715         class D(dict):
 716             def __getitem__(self, key):
 717                 raise ValueError
 718         self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
 719         self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
 720         self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
 721
 722     def test_encodehelper(self):
 723         # enhance coverage of:
 724         # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
 725         # and callers
 726         self.assertRaises(LookupError, u"\xff".encode, "ascii", "test.unknown")
 727
 728         def badencodereturn1(exc):
 729             return 42
 730         codecs.register_error("test.badencodereturn1", badencodereturn1)
 731         self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn1")
 732
 733         def badencodereturn2(exc):
 734             return (u"?", None)
 735         codecs.register_error("test.badencodereturn2", badencodereturn2)
 736         self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2")
 737
 738         handler = PosReturn()
 739         codecs.register_error("test.posreturn", handler.handle)
 740
 741         # Valid negative position
 742         handler.pos = -1
 743         self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
 744
 745         # Valid negative position
 746         handler.pos = -2
 747         self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?><?>")
 748
 749         # Negative position out of bounds
 750         handler.pos = -3
 751         self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
 752
 753         # Valid positive position
 754         handler.pos = 1
 755         self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
 756
 757         # Largest valid positive position (one beyond end of input
 758         handler.pos = 2
 759         self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>")
 760
 761         # Invalid positive position
 762         handler.pos = 3
 763         self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
 764
 765         handler.pos = 0
 766
 767         class D(dict):
 768             def __getitem__(self, key):
 769                 raise ValueError
 770         for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"):
 771             self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None})
 772             self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D())
 773             self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300})
 774
 775     def test_translatehelper(self):
 776         # enhance coverage of:
 777         # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
 778         # and callers
 779         # (Unfortunately the errors argument is not directly accessible
 780         # from Python, so we can't test that much)
 781         class D(dict):
 782             def __getitem__(self, key):
 783                 raise ValueError
 784         self.assertRaises(ValueError, u"\xff".translate, D())
 785         self.assertRaises(TypeError, u"\xff".translate, {0xff: sys.maxunicode+1})
 786         self.assertRaises(TypeError, u"\xff".translate, {0xff: ()})
 787
 788     def test_bug828737(self):
 789         charmap = {
 790             ord("&"): u"&amp;",
 791             ord("<"): u"&lt;",
 792             ord(">"): u"&gt;",
 793             ord('"'): u"&quot;",
 794         }
 795
 796         for n in (1, 10, 100, 1000):
 797             text = u'abc<def>ghi'*n
 798             text.translate(charmap)
 799
 800 def test_main():
 801     test.test_support.run_unittest(CodecCallbackTest)
 802
 803 if __name__ == "__main__":
 804     test_main()