Lib/test/test_multibytecodec.py

   1 #!/usr/bin/env python
   2 #
   3 # test_multibytecodec.py
   4 #   Unit test for multibytecodec itself
   5 #
   6
   7 from test import test_support
   8 from test import test_multibytecodec_support
   9 from test.test_support import TESTFN
  10 import unittest, StringIO, codecs, sys, os
  11 import _multibytecodec
  12
  13 ALL_CJKENCODINGS = [
  14 # _codecs_cn
  15     'gb2312', 'gbk', 'gb18030', 'hz',
  16 # _codecs_hk
  17     'big5hkscs',
  18 # _codecs_jp
  19     'cp932', 'shift_jis', 'euc_jp', 'euc_jisx0213', 'shift_jisx0213',
  20     'euc_jis_2004', 'shift_jis_2004',
  21 # _codecs_kr
  22     'cp949', 'euc_kr', 'johab',
  23 # _codecs_tw
  24     'big5', 'cp950',
  25 # _codecs_iso2022
  26     'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2', 'iso2022_jp_2004',
  27     'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr',
  28 ]
  29
  30 class Test_MultibyteCodec(unittest.TestCase):
  31
  32     def test_nullcoding(self):
  33         for enc in ALL_CJKENCODINGS:
  34             self.assertEqual(''.decode(enc), u'')
  35             self.assertEqual(unicode('', enc), u'')
  36             self.assertEqual(u''.encode(enc), '')
  37
  38     def test_str_decode(self):
  39         for enc in ALL_CJKENCODINGS:
  40             self.assertEqual('abcd'.encode(enc), 'abcd')
  41
  42     def test_errorcallback_longindex(self):
  43         dec = codecs.getdecoder('euc-kr')
  44         myreplace  = lambda exc: (u'', sys.maxint+1)
  45         codecs.register_error('test.cjktest', myreplace)
  46         self.assertRaises(IndexError, dec,
  47                           'apple\x92ham\x93spam', 'test.cjktest')
  48
  49     def test_codingspec(self):
  50         try:
  51             for enc in ALL_CJKENCODINGS:
  52                 print >> open(TESTFN, 'w'), '# coding:', enc
  53                 exec open(TESTFN)
  54         finally:
  55             os.unlink(TESTFN)
  56
  57     def test_init_segfault(self):
  58         # bug #3305: this used to segfault
  59         self.assertRaises(AttributeError,
  60                           _multibytecodec.MultibyteStreamReader, None)
  61         self.assertRaises(AttributeError,
  62                           _multibytecodec.MultibyteStreamWriter, None)
  63
  64
  65 class Test_IncrementalEncoder(unittest.TestCase):
  66
  67     def test_stateless(self):
  68         # cp949 encoder isn't stateful at all.
  69         encoder = codecs.getincrementalencoder('cp949')()
  70         self.assertEqual(encoder.encode(u'\ud30c\uc774\uc36c \ub9c8\uc744'),
  71                          '\xc6\xc4\xc0\xcc\xbd\xe3 \xb8\xb6\xc0\xbb')
  72         self.assertEqual(encoder.reset(), None)
  73         self.assertEqual(encoder.encode(u'\u2606\u223c\u2606', True),
  74                          '\xa1\xd9\xa1\xad\xa1\xd9')
  75         self.assertEqual(encoder.reset(), None)
  76         self.assertEqual(encoder.encode(u'', True), '')
  77         self.assertEqual(encoder.encode(u'', False), '')
  78         self.assertEqual(encoder.reset(), None)
  79
  80     def test_stateful(self):
  81         # jisx0213 encoder is stateful for a few codepoints. eg)
  82         #   U+00E6 => A9DC
  83         #   U+00E6 U+0300 => ABC4
  84         #   U+0300 => ABDC
  85
  86         encoder = codecs.getincrementalencoder('jisx0213')()
  87         self.assertEqual(encoder.encode(u'\u00e6\u0300'), '\xab\xc4')
  88         self.assertEqual(encoder.encode(u'\u00e6'), '')
  89         self.assertEqual(encoder.encode(u'\u0300'), '\xab\xc4')
  90         self.assertEqual(encoder.encode(u'\u00e6', True), '\xa9\xdc')
  91
  92         self.assertEqual(encoder.reset(), None)
  93         self.assertEqual(encoder.encode(u'\u0300'), '\xab\xdc')
  94
  95         self.assertEqual(encoder.encode(u'\u00e6'), '')
  96         self.assertEqual(encoder.encode('', True), '\xa9\xdc')
  97         self.assertEqual(encoder.encode('', True), '')
  98
  99     def test_stateful_keep_buffer(self):
 100         encoder = codecs.getincrementalencoder('jisx0213')()
 101         self.assertEqual(encoder.encode(u'\u00e6'), '')
 102         self.assertRaises(UnicodeEncodeError, encoder.encode, u'\u0123')
 103         self.assertEqual(encoder.encode(u'\u0300\u00e6'), '\xab\xc4')
 104         self.assertRaises(UnicodeEncodeError, encoder.encode, u'\u0123')
 105         self.assertEqual(encoder.reset(), None)
 106         self.assertEqual(encoder.encode(u'\u0300'), '\xab\xdc')
 107         self.assertEqual(encoder.encode(u'\u00e6'), '')
 108         self.assertRaises(UnicodeEncodeError, encoder.encode, u'\u0123')
 109         self.assertEqual(encoder.encode(u'', True), '\xa9\xdc')
 110
 111
 112 class Test_IncrementalDecoder(unittest.TestCase):
 113
 114     def test_dbcs(self):
 115         # cp949 decoder is simple with only 1 or 2 bytes sequences.
 116         decoder = codecs.getincrementaldecoder('cp949')()
 117         self.assertEqual(decoder.decode('\xc6\xc4\xc0\xcc\xbd'),
 118                          u'\ud30c\uc774')
 119         self.assertEqual(decoder.decode('\xe3 \xb8\xb6\xc0\xbb'),
 120                          u'\uc36c \ub9c8\uc744')
 121         self.assertEqual(decoder.decode(''), u'')
 122
 123     def test_dbcs_keep_buffer(self):
 124         decoder = codecs.getincrementaldecoder('cp949')()
 125         self.assertEqual(decoder.decode('\xc6\xc4\xc0'), u'\ud30c')
 126         self.assertRaises(UnicodeDecodeError, decoder.decode, '', True)
 127         self.assertEqual(decoder.decode('\xcc'), u'\uc774')
 128
 129         self.assertEqual(decoder.decode('\xc6\xc4\xc0'), u'\ud30c')
 130         self.assertRaises(UnicodeDecodeError, decoder.decode, '\xcc\xbd', True)
 131         self.assertEqual(decoder.decode('\xcc'), u'\uc774')
 132
 133     def test_iso2022(self):
 134         decoder = codecs.getincrementaldecoder('iso2022-jp')()
 135         ESC = '\x1b'
 136         self.assertEqual(decoder.decode(ESC + '('), u'')
 137         self.assertEqual(decoder.decode('B', True), u'')
 138         self.assertEqual(decoder.decode(ESC + '$'), u'')
 139         self.assertEqual(decoder.decode('B@$'), u'\u4e16')
 140         self.assertEqual(decoder.decode('@$@'), u'\u4e16')
 141         self.assertEqual(decoder.decode('$', True), u'\u4e16')
 142         self.assertEqual(decoder.reset(), None)
 143         self.assertEqual(decoder.decode('@$'), u'@$')
 144         self.assertEqual(decoder.decode(ESC + '$'), u'')
 145         self.assertRaises(UnicodeDecodeError, decoder.decode, '', True)
 146         self.assertEqual(decoder.decode('B@$'), u'\u4e16')
 147
 148 class Test_StreamReader(unittest.TestCase):
 149     def test_bug1728403(self):
 150         try:
 151             open(TESTFN, 'w').write('\xa1')
 152             f = codecs.open(TESTFN, encoding='cp949')
 153             self.assertRaises(UnicodeDecodeError, f.read, 2)
 154         finally:
 155             try: f.close()
 156             except: pass
 157             os.unlink(TESTFN)
 158
 159 class Test_StreamWriter(unittest.TestCase):
 160     if len(u'\U00012345') == 2: # UCS2
 161         def test_gb18030(self):
 162             s = StringIO.StringIO()
 163             c = codecs.getwriter('gb18030')(s)
 164             c.write(u'123')
 165             self.assertEqual(s.getvalue(), '123')
 166             c.write(u'\U00012345')
 167             self.assertEqual(s.getvalue(), '123\x907\x959')
 168             c.write(u'\U00012345'[0])
 169             self.assertEqual(s.getvalue(), '123\x907\x959')
 170             c.write(u'\U00012345'[1] + u'\U00012345' + u'\uac00\u00ac')
 171             self.assertEqual(s.getvalue(),
 172                     '123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
 173             c.write(u'\U00012345'[0])
 174             self.assertEqual(s.getvalue(),
 175                     '123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
 176             self.assertRaises(UnicodeError, c.reset)
 177             self.assertEqual(s.getvalue(),
 178                     '123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
 179
 180         def test_utf_8(self):
 181             s= StringIO.StringIO()
 182             c = codecs.getwriter('utf-8')(s)
 183             c.write(u'123')
 184             self.assertEqual(s.getvalue(), '123')
 185             c.write(u'\U00012345')
 186             self.assertEqual(s.getvalue(), '123\xf0\x92\x8d\x85')
 187
 188             # Python utf-8 codec can't buffer surrogate pairs yet.
 189             if 0:
 190                 c.write(u'\U00012345'[0])
 191                 self.assertEqual(s.getvalue(), '123\xf0\x92\x8d\x85')
 192                 c.write(u'\U00012345'[1] + u'\U00012345' + u'\uac00\u00ac')
 193                 self.assertEqual(s.getvalue(),
 194                     '123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
 195                     '\xea\xb0\x80\xc2\xac')
 196                 c.write(u'\U00012345'[0])
 197                 self.assertEqual(s.getvalue(),
 198                     '123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
 199                     '\xea\xb0\x80\xc2\xac')
 200                 c.reset()
 201                 self.assertEqual(s.getvalue(),
 202                     '123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
 203                     '\xea\xb0\x80\xc2\xac\xed\xa0\x88')
 204                 c.write(u'\U00012345'[1])
 205                 self.assertEqual(s.getvalue(),
 206                     '123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
 207                     '\xea\xb0\x80\xc2\xac\xed\xa0\x88\xed\xbd\x85')
 208
 209     else: # UCS4
 210         pass
 211
 212     def test_streamwriter_strwrite(self):
 213         s = StringIO.StringIO()
 214         wr = codecs.getwriter('gb18030')(s)
 215         wr.write('abcd')
 216         self.assertEqual(s.getvalue(), 'abcd')
 217
 218 class Test_ISO2022(unittest.TestCase):
 219     def test_g2(self):
 220         iso2022jp2 = '\x1b(B:hu4:unit\x1b.A\x1bNi de famille'
 221         uni = u':hu4:unit\xe9 de famille'
 222         self.assertEqual(iso2022jp2.decode('iso2022-jp-2'), uni)
 223
 224     def test_iso2022_jp_g0(self):
 225         self.assertFalse('\x0e' in u'\N{SOFT HYPHEN}'.encode('iso-2022-jp-2'))
 226         for encoding in ('iso-2022-jp-2004', 'iso-2022-jp-3'):
 227             e = u'\u3406'.encode(encoding)
 228             self.assertFalse(filter(lambda x: x >= '\x80', e))
 229
 230     def test_bug1572832(self):
 231         if sys.maxunicode >= 0x10000:
 232             myunichr = unichr
 233         else:
 234             myunichr = lambda x: unichr(0xD7C0+(x>>10)) + unichr(0xDC00+(x&0x3FF))
 235
 236         for x in xrange(0x10000, 0x110000):
 237             # Any ISO 2022 codec will cause the segfault
 238             myunichr(x).encode('iso_2022_jp', 'ignore')
 239
 240 def test_main():
 241     test_support.run_unittest(__name__)
 242
 243 if __name__ == "__main__":
 244     test_main()