Lib/test/test_struct.py

   1 import array
   2 import unittest
   3 import struct
   4 import warnings
   5 warnings.filterwarnings("ignore", "struct integer overflow masking is deprecated",
   6                         DeprecationWarning)
   7
   8 from functools import wraps
   9 from test.test_support import TestFailed, verbose, run_unittest
  10
  11 import sys
  12 ISBIGENDIAN = sys.byteorder == "big"
  13 IS32BIT = sys.maxsize == 0x7fffffff
  14
  15 integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q'
  16
  17 # Native 'q' packing isn't available on systems that don't have the C
  18 # long long type.
  19 try:
  20     struct.pack('q', 5)
  21 except struct.error:
  22     HAVE_LONG_LONG = False
  23 else:
  24     HAVE_LONG_LONG = True
  25
  26 try:
  27     import _struct
  28 except ImportError:
  29     PY_STRUCT_FLOAT_COERCE = 2
  30 else:
  31     PY_STRUCT_FLOAT_COERCE = getattr(_struct, '_PY_STRUCT_FLOAT_COERCE', 0)
  32
  33 def string_reverse(s):
  34     return "".join(reversed(s))
  35
  36 def bigendian_to_native(value):
  37     if ISBIGENDIAN:
  38         return value
  39     else:
  40         return string_reverse(value)
  41
  42 def with_warning_restore(func):
  43     @wraps(func)
  44     def decorator(*args, **kw):
  45         with warnings.catch_warnings():
  46             # We need this function to warn every time, so stick an
  47             # unqualifed 'always' at the head of the filter list
  48             warnings.simplefilter("always")
  49             warnings.filterwarnings("error", category=DeprecationWarning)
  50             return func(*args, **kw)
  51     return decorator
  52
  53 class StructTest(unittest.TestCase):
  54
  55     @with_warning_restore
  56     def check_float_coerce(self, format, number):
  57         # SF bug 1530559. struct.pack raises TypeError where it used to convert.
  58         if PY_STRUCT_FLOAT_COERCE == 2:
  59             # Test for pre-2.5 struct module
  60             packed = struct.pack(format, number)
  61             floored = struct.unpack(format, packed)[0]
  62             self.assertEqual(floored, int(number),
  63                              "did not correcly coerce float to int")
  64             return
  65         try:
  66             struct.pack(format, number)
  67         except struct.error:
  68             if PY_STRUCT_FLOAT_COERCE:
  69                 self.fail("expected DeprecationWarning for float coerce")
  70         except DeprecationWarning:
  71             if not PY_STRUCT_FLOAT_COERCE:
  72                 self.fail("expected to raise struct.error for float coerce")
  73         else:
  74             self.fail("did not raise error for float coerce")
  75
  76     def test_isbigendian(self):
  77         self.assertEqual((struct.pack('=i', 1)[0] == chr(0)), ISBIGENDIAN)
  78
  79     def test_consistence(self):
  80         self.assertRaises(struct.error, struct.calcsize, 'Z')
  81
  82         sz = struct.calcsize('i')
  83         self.assertEqual(sz * 3, struct.calcsize('iii'))
  84
  85         fmt = 'cbxxxxxxhhhhiillffd?'
  86         fmt3 = '3c3b18x12h6i6l6f3d3?'
  87         sz = struct.calcsize(fmt)
  88         sz3 = struct.calcsize(fmt3)
  89         self.assertEqual(sz * 3, sz3)
  90
  91         self.assertRaises(struct.error, struct.pack, 'iii', 3)
  92         self.assertRaises(struct.error, struct.pack, 'i', 3, 3, 3)
  93         self.assertRaises(struct.error, struct.pack, 'i', 'foo')
  94         self.assertRaises(struct.error, struct.pack, 'P', 'foo')
  95         self.assertRaises(struct.error, struct.unpack, 'd', 'flap')
  96         s = struct.pack('ii', 1, 2)
  97         self.assertRaises(struct.error, struct.unpack, 'iii', s)
  98         self.assertRaises(struct.error, struct.unpack, 'i', s)
  99
 100     def test_transitiveness(self):
 101         c = 'a'
 102         b = 1
 103         h = 255
 104         i = 65535
 105         l = 65536
 106         f = 3.1415
 107         d = 3.1415
 108         t = True
 109
 110         for prefix in ('', '@', '<', '>', '=', '!'):
 111             for format in ('xcbhilfd?', 'xcBHILfd?'):
 112                 format = prefix + format
 113                 s = struct.pack(format, c, b, h, i, l, f, d, t)
 114                 cp, bp, hp, ip, lp, fp, dp, tp = struct.unpack(format, s)
 115                 self.assertEqual(cp, c)
 116                 self.assertEqual(bp, b)
 117                 self.assertEqual(hp, h)
 118                 self.assertEqual(ip, i)
 119                 self.assertEqual(lp, l)
 120                 self.assertEqual(int(100 * fp), int(100 * f))
 121                 self.assertEqual(int(100 * dp), int(100 * d))
 122                 self.assertEqual(tp, t)
 123
 124     def test_new_features(self):
 125         # Test some of the new features in detail
 126         # (format, argument, big-endian result, little-endian result, asymmetric)
 127         tests = [
 128             ('c', 'a', 'a', 'a', 0),
 129             ('xc', 'a', '\0a', '\0a', 0),
 130             ('cx', 'a', 'a\0', 'a\0', 0),
 131             ('s', 'a', 'a', 'a', 0),
 132             ('0s', 'helloworld', '', '', 1),
 133             ('1s', 'helloworld', 'h', 'h', 1),
 134             ('9s', 'helloworld', 'helloworl', 'helloworl', 1),
 135             ('10s', 'helloworld', 'helloworld', 'helloworld', 0),
 136             ('11s', 'helloworld', 'helloworld\0', 'helloworld\0', 1),
 137             ('20s', 'helloworld', 'helloworld'+10*'\0', 'helloworld'+10*'\0', 1),
 138             ('b', 7, '\7', '\7', 0),
 139             ('b', -7, '\371', '\371', 0),
 140             ('B', 7, '\7', '\7', 0),
 141             ('B', 249, '\371', '\371', 0),
 142             ('h', 700, '\002\274', '\274\002', 0),
 143             ('h', -700, '\375D', 'D\375', 0),
 144             ('H', 700, '\002\274', '\274\002', 0),
 145             ('H', 0x10000-700, '\375D', 'D\375', 0),
 146             ('i', 70000000, '\004,\035\200', '\200\035,\004', 0),
 147             ('i', -70000000, '\373\323\342\200', '\200\342\323\373', 0),
 148             ('I', 70000000L, '\004,\035\200', '\200\035,\004', 0),
 149             ('I', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0),
 150             ('l', 70000000, '\004,\035\200', '\200\035,\004', 0),
 151             ('l', -70000000, '\373\323\342\200', '\200\342\323\373', 0),
 152             ('L', 70000000L, '\004,\035\200', '\200\035,\004', 0),
 153             ('L', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0),
 154             ('f', 2.0, '@\000\000\000', '\000\000\000@', 0),
 155             ('d', 2.0, '@\000\000\000\000\000\000\000',
 156                        '\000\000\000\000\000\000\000@', 0),
 157             ('f', -2.0, '\300\000\000\000', '\000\000\000\300', 0),
 158             ('d', -2.0, '\300\000\000\000\000\000\000\000',
 159                        '\000\000\000\000\000\000\000\300', 0),
 160                 ('?', 0, '\0', '\0', 0),
 161                 ('?', 3, '\1', '\1', 1),
 162                 ('?', True, '\1', '\1', 0),
 163                 ('?', [], '\0', '\0', 1),
 164                 ('?', (1,), '\1', '\1', 1),
 165         ]
 166
 167         for fmt, arg, big, lil, asy in tests:
 168             for (xfmt, exp) in [('>'+fmt, big), ('!'+fmt, big), ('<'+fmt, lil),
 169                                 ('='+fmt, ISBIGENDIAN and big or lil)]:
 170                 res = struct.pack(xfmt, arg)
 171                 self.assertEqual(res, exp)
 172                 self.assertEqual(struct.calcsize(xfmt), len(res))
 173                 rev = struct.unpack(xfmt, res)[0]
 174                 if rev != arg:
 175                     self.assertTrue(asy)
 176
 177     def test_calcsize(self):
 178         expected_size = {
 179             'b': 1, 'B': 1,
 180             'h': 2, 'H': 2,
 181             'i': 4, 'I': 4,
 182             'l': 4, 'L': 4,
 183             'q': 8, 'Q': 8,
 184             }
 185
 186         # standard integer sizes
 187         for code in integer_codes:
 188             for byteorder in ('=', '<', '>', '!'):
 189                 format = byteorder+code
 190                 size = struct.calcsize(format)
 191                 self.assertEqual(size, expected_size[code])
 192
 193         # native integer sizes, except 'q' and 'Q'
 194         for format_pair in ('bB', 'hH', 'iI', 'lL'):
 195             for byteorder in ['', '@']:
 196                 signed_size = struct.calcsize(byteorder + format_pair[0])
 197                 unsigned_size = struct.calcsize(byteorder + format_pair[1])
 198                 self.assertEqual(signed_size, unsigned_size)
 199
 200         # bounds for native integer sizes
 201         self.assertTrue(struct.calcsize('b')==1)
 202         self.assertTrue(2 <= struct.calcsize('h'))
 203         self.assertTrue(4 <= struct.calcsize('l'))
 204         self.assertTrue(struct.calcsize('h') <= struct.calcsize('i'))
 205         self.assertTrue(struct.calcsize('i') <= struct.calcsize('l'))
 206
 207         # tests for native 'q' and 'Q' when applicable
 208         if HAVE_LONG_LONG:
 209             self.assertEqual(struct.calcsize('q'), struct.calcsize('Q'))
 210             self.assertTrue(8 <= struct.calcsize('q'))
 211             self.assertTrue(struct.calcsize('l') <= struct.calcsize('q'))
 212
 213     def test_integers(self):
 214         # Integer tests (bBhHiIlLqQ).
 215         import binascii
 216
 217         class IntTester(unittest.TestCase):
 218             def __init__(self, format):
 219                 super(IntTester, self).__init__(methodName='test_one')
 220                 self.format = format
 221                 self.code = format[-1]
 222                 self.direction = format[:-1]
 223                 if not self.direction in ('', '@', '=', '<', '>', '!'):
 224                     raise ValueError("unrecognized packing direction: %s" %
 225                                      self.direction)
 226                 self.bytesize = struct.calcsize(format)
 227                 self.bitsize = self.bytesize * 8
 228                 if self.code in tuple('bhilq'):
 229                     self.signed = True
 230                     self.min_value = -(2L**(self.bitsize-1))
 231                     self.max_value = 2L**(self.bitsize-1) - 1
 232                 elif self.code in tuple('BHILQ'):
 233                     self.signed = False
 234                     self.min_value = 0
 235                     self.max_value = 2L**self.bitsize - 1
 236                 else:
 237                     raise ValueError("unrecognized format code: %s" %
 238                                      self.code)
 239
 240             def test_one(self, x, pack=struct.pack,
 241                                   unpack=struct.unpack,
 242                                   unhexlify=binascii.unhexlify):
 243
 244                 format = self.format
 245                 if self.min_value <= x <= self.max_value:
 246                     expected = long(x)
 247                     if self.signed and x < 0:
 248                         expected += 1L << self.bitsize
 249                     self.assertTrue(expected >= 0)
 250                     expected = '%x' % expected
 251                     if len(expected) & 1:
 252                         expected = "0" + expected
 253                     expected = unhexlify(expected)
 254                     expected = ("\x00" * (self.bytesize - len(expected)) +
 255                                 expected)
 256                     if (self.direction == '<' or
 257                         self.direction in ('', '@', '=') and not ISBIGENDIAN):
 258                         expected = string_reverse(expected)
 259                     self.assertEqual(len(expected), self.bytesize)
 260
 261                     # Pack work?
 262                     got = pack(format, x)
 263                     self.assertEqual(got, expected)
 264
 265                     # Unpack work?
 266                     retrieved = unpack(format, got)[0]
 267                     self.assertEqual(x, retrieved)
 268
 269                     # Adding any byte should cause a "too big" error.
 270                     self.assertRaises((struct.error, TypeError), unpack, format,
 271                                                                  '\x01' + got)
 272                 else:
 273                     # x is out of range -- verify pack realizes that.
 274                     self.assertRaises(struct.error, pack, format, x)
 275
 276             def run(self):
 277                 from random import randrange
 278
 279                 # Create all interesting powers of 2.
 280                 values = []
 281                 for exp in range(self.bitsize + 3):
 282                     values.append(1L << exp)
 283
 284                 # Add some random values.
 285                 for i in range(self.bitsize):
 286                     val = 0L
 287                     for j in range(self.bytesize):
 288                         val = (val << 8) | randrange(256)
 289                     values.append(val)
 290
 291                 # Values absorbed from other tests
 292                 values.extend([300, 700000, sys.maxint*4])
 293
 294                 # Try all those, and their negations, and +-1 from
 295                 # them.  Note that this tests all power-of-2
 296                 # boundaries in range, and a few out of range, plus
 297                 # +-(2**n +- 1).
 298                 for base in values:
 299                     for val in -base, base:
 300                         for incr in -1, 0, 1:
 301                             x = val + incr
 302                             self.test_one(int(x))
 303                             self.test_one(long(x))
 304
 305                 # Some error cases.
 306                 class NotAnIntNS(object):
 307                     def __int__(self):
 308                         return 42
 309
 310                     def __long__(self):
 311                         return 1729L
 312
 313                 class NotAnIntOS:
 314                     def __int__(self):
 315                         return 10585
 316
 317                     def __long__(self):
 318                         return -163L
 319
 320                 for badobject in ("a string", 3+42j, randrange,
 321                                   NotAnIntNS(), NotAnIntOS()):
 322                     self.assertRaises(struct.error,
 323                                       struct.pack, format,
 324                                       badobject)
 325
 326         byteorders = '', '@', '=', '<', '>', '!'
 327         for code in integer_codes:
 328             for byteorder in byteorders:
 329                 if (byteorder in ('', '@') and code in ('q', 'Q') and
 330                     not HAVE_LONG_LONG):
 331                     continue
 332                 format = byteorder+code
 333                 t = IntTester(format)
 334                 t.run()
 335
 336     def test_p_code(self):
 337         # Test p ("Pascal string") code.
 338         for code, input, expected, expectedback in [
 339                 ('p','abc', '\x00', ''),
 340                 ('1p', 'abc', '\x00', ''),
 341                 ('2p', 'abc', '\x01a', 'a'),
 342                 ('3p', 'abc', '\x02ab', 'ab'),
 343                 ('4p', 'abc', '\x03abc', 'abc'),
 344                 ('5p', 'abc', '\x03abc\x00', 'abc'),
 345                 ('6p', 'abc', '\x03abc\x00\x00', 'abc'),
 346                 ('1000p', 'x'*1000, '\xff' + 'x'*999, 'x'*255)]:
 347             got = struct.pack(code, input)
 348             self.assertEqual(got, expected)
 349             (got,) = struct.unpack(code, got)
 350             self.assertEqual(got, expectedback)
 351
 352     def test_705836(self):
 353         # SF bug 705836.  "<f" and ">f" had a severe rounding bug, where a carry
 354         # from the low-order discarded bits could propagate into the exponent
 355         # field, causing the result to be wrong by a factor of 2.
 356         import math
 357
 358         for base in range(1, 33):
 359             # smaller <- largest representable float less than base.
 360             delta = 0.5
 361             while base - delta / 2.0 != base:
 362                 delta /= 2.0
 363             smaller = base - delta
 364             # Packing this rounds away a solid string of trailing 1 bits.
 365             packed = struct.pack("<f", smaller)
 366             unpacked = struct.unpack("<f", packed)[0]
 367             # This failed at base = 2, 4, and 32, with unpacked = 1, 2, and
 368             # 16, respectively.
 369             self.assertEqual(base, unpacked)
 370             bigpacked = struct.pack(">f", smaller)
 371             self.assertEqual(bigpacked, string_reverse(packed))
 372             unpacked = struct.unpack(">f", bigpacked)[0]
 373             self.assertEqual(base, unpacked)
 374
 375         # Largest finite IEEE single.
 376         big = (1 << 24) - 1
 377         big = math.ldexp(big, 127 - 23)
 378         packed = struct.pack(">f", big)
 379         unpacked = struct.unpack(">f", packed)[0]
 380         self.assertEqual(big, unpacked)
 381
 382         # The same, but tack on a 1 bit so it rounds up to infinity.
 383         big = (1 << 25) - 1
 384         big = math.ldexp(big, 127 - 24)
 385         self.assertRaises(OverflowError, struct.pack, ">f", big)
 386
 387     def test_1530559(self):
 388         # SF bug 1530559. struct.pack raises TypeError where it used to convert.
 389         for endian in ('', '>', '<'):
 390             for fmt in integer_codes:
 391                 self.check_float_coerce(endian + fmt, 1.0)
 392                 self.check_float_coerce(endian + fmt, 1.5)
 393
 394     def test_unpack_from(self):
 395         test_string = 'abcd01234'
 396         fmt = '4s'
 397         s = struct.Struct(fmt)
 398         for cls in (str, buffer):
 399             data = cls(test_string)
 400             self.assertEqual(s.unpack_from(data), ('abcd',))
 401             self.assertEqual(s.unpack_from(data, 2), ('cd01',))
 402             self.assertEqual(s.unpack_from(data, 4), ('0123',))
 403             for i in xrange(6):
 404                 self.assertEqual(s.unpack_from(data, i), (data[i:i+4],))
 405             for i in xrange(6, len(test_string) + 1):
 406                 self.assertRaises(struct.error, s.unpack_from, data, i)
 407         for cls in (str, buffer):
 408             data = cls(test_string)
 409             self.assertEqual(struct.unpack_from(fmt, data), ('abcd',))
 410             self.assertEqual(struct.unpack_from(fmt, data, 2), ('cd01',))
 411             self.assertEqual(struct.unpack_from(fmt, data, 4), ('0123',))
 412             for i in xrange(6):
 413                 self.assertEqual(struct.unpack_from(fmt, data, i), (data[i:i+4],))
 414             for i in xrange(6, len(test_string) + 1):
 415                 self.assertRaises(struct.error, struct.unpack_from, fmt, data, i)
 416
 417     def test_pack_into(self):
 418         test_string = 'Reykjavik rocks, eow!'
 419         writable_buf = array.array('c', ' '*100)
 420         fmt = '21s'
 421         s = struct.Struct(fmt)
 422
 423         # Test without offset
 424         s.pack_into(writable_buf, 0, test_string)
 425         from_buf = writable_buf.tostring()[:len(test_string)]
 426         self.assertEqual(from_buf, test_string)
 427
 428         # Test with offset.
 429         s.pack_into(writable_buf, 10, test_string)
 430         from_buf = writable_buf.tostring()[:len(test_string)+10]
 431         self.assertEqual(from_buf, test_string[:10] + test_string)
 432
 433         # Go beyond boundaries.
 434         small_buf = array.array('c', ' '*10)
 435         self.assertRaises(struct.error, s.pack_into, small_buf, 0, test_string)
 436         self.assertRaises(struct.error, s.pack_into, small_buf, 2, test_string)
 437
 438         # Test bogus offset (issue 3694)
 439         sb = small_buf
 440         self.assertRaises(TypeError, struct.pack_into, b'1', sb, None)
 441
 442     def test_pack_into_fn(self):
 443         test_string = 'Reykjavik rocks, eow!'
 444         writable_buf = array.array('c', ' '*100)
 445         fmt = '21s'
 446         pack_into = lambda *args: struct.pack_into(fmt, *args)
 447
 448         # Test without offset.
 449         pack_into(writable_buf, 0, test_string)
 450         from_buf = writable_buf.tostring()[:len(test_string)]
 451         self.assertEqual(from_buf, test_string)
 452
 453         # Test with offset.
 454         pack_into(writable_buf, 10, test_string)
 455         from_buf = writable_buf.tostring()[:len(test_string)+10]
 456         self.assertEqual(from_buf, test_string[:10] + test_string)
 457
 458         # Go beyond boundaries.
 459         small_buf = array.array('c', ' '*10)
 460         self.assertRaises(struct.error, pack_into, small_buf, 0, test_string)
 461         self.assertRaises(struct.error, pack_into, small_buf, 2, test_string)
 462
 463     def test_unpack_with_buffer(self):
 464         # SF bug 1563759: struct.unpack doens't support buffer protocol objects
 465         data1 = array.array('B', '\x12\x34\x56\x78')
 466         data2 = buffer('......\x12\x34\x56\x78......', 6, 4)
 467         for data in [data1, data2]:
 468             value, = struct.unpack('>I', data)
 469             self.assertEqual(value, 0x12345678)
 470
 471     def test_bool(self):
 472         for prefix in tuple("<>!=")+('',):
 473             false = (), [], [], '', 0
 474             true = [1], 'test', 5, -1, 0xffffffffL+1, 0xffffffff/2
 475
 476             falseFormat = prefix + '?' * len(false)
 477             packedFalse = struct.pack(falseFormat, *false)
 478             unpackedFalse = struct.unpack(falseFormat, packedFalse)
 479
 480             trueFormat = prefix + '?' * len(true)
 481             packedTrue = struct.pack(trueFormat, *true)
 482             unpackedTrue = struct.unpack(trueFormat, packedTrue)
 483
 484             self.assertEqual(len(true), len(unpackedTrue))
 485             self.assertEqual(len(false), len(unpackedFalse))
 486
 487             for t in unpackedFalse:
 488                 self.assertFalse(t)
 489             for t in unpackedTrue:
 490                 self.assertTrue(t)
 491
 492             packed = struct.pack(prefix+'?', 1)
 493
 494             self.assertEqual(len(packed), struct.calcsize(prefix+'?'))
 495
 496             if len(packed) != 1:
 497                 self.assertFalse(prefix, msg='encoded bool is not one byte: %r'
 498                                              %packed)
 499
 500             for c in '\x01\x7f\xff\x0f\xf0':
 501                 self.assertTrue(struct.unpack('>?', c)[0])
 502
 503     if IS32BIT:
 504         def test_crasher(self):
 505             self.assertRaises(MemoryError, struct.pack, "357913941c", "a")
 506
 507
 508
 509 def test_main():
 510     run_unittest(StructTest)
 511
 512 if __name__ == '__main__':
 513     test_main()