lib/py/src/protocol/fastbinary.c

   1 // Copyright (c) 2006- Facebook
   2 // Distributed under the Thrift Software License
   3 //
   4 // See accompanying file LICENSE or visit the Thrift site at:
   5 // http://developers.facebook.com/thrift/
   6 //
   7 // NOTE:  This code was contributed by an external developer.
   8 //        The internal Thrift team has reviewed and tested it,
   9 //        but we cannot guarantee that it is production-ready.
  10 //        Please feel free to report bugs and/or success stories
  11 //        to the public mailing list.
  12
  13 #include <Python.h>
  14 #include "cStringIO.h"
  15 #include <stdbool.h>
  16 #include <stdint.h>
  17 #include <netinet/in.h>
  18
  19 // TODO(dreiss): defval appears to be unused.  Look into removing it.
  20 // TODO(dreiss): Make parse_spec_args recursive, and cache the output
  21 //               permanently in the object.  (Malloc and orphan.)
  22 // TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
  23 //               Can cStringIO let us work with a BufferedTransport?
  24 // TODO(dreiss): Don't ignore the rv from cwrite (maybe).
  25
  26 /* ====== BEGIN UTILITIES ====== */
  27
  28 #define INIT_OUTBUF_SIZE 128
  29
  30 // Stolen out of TProtocol.h.
  31 // It would be a huge pain to have both get this from one place.
  32 typedef enum TType {
  33   T_STOP       = 0,
  34   T_VOID       = 1,
  35   T_BOOL       = 2,
  36   T_BYTE       = 3,
  37   T_I08        = 3,
  38   T_I16        = 6,
  39   T_I32        = 8,
  40   T_U64        = 9,
  41   T_I64        = 10,
  42   T_DOUBLE     = 4,
  43   T_STRING     = 11,
  44   T_UTF7       = 11,
  45   T_STRUCT     = 12,
  46   T_MAP        = 13,
  47   T_SET        = 14,
  48   T_LIST       = 15,
  49   T_UTF8       = 16,
  50   T_UTF16      = 17
  51 } TType;
  52
  53 // Same comment as the enum.  Sorry.
  54 #if __BYTE_ORDER == __BIG_ENDIAN
  55 # define ntohll(n) (n)
  56 # define htonll(n) (n)
  57 #elif __BYTE_ORDER == __LITTLE_ENDIAN
  58 # if defined(__GNUC__) && defined(__GLIBC__)
  59 #  include <byteswap.h>
  60 #  define ntohll(n) bswap_64(n)
  61 #  define htonll(n) bswap_64(n)
  62 # else /* GNUC & GLIBC */
  63 #  define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) )
  64 #  define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) )
  65 # endif /* GNUC & GLIBC */
  66 #else /* __BYTE_ORDER */
  67 # error "Can't define htonll or ntohll!"
  68 #endif
  69
  70 // Doing a benchmark shows that interning actually makes a difference, amazingly.
  71 #define INTERN_STRING(value) _intern_ ## value
  72
  73 #define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() )
  74 #define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) )
  75
  76 // Py_ssize_t was not defined before Python 2.5
  77 #if (PY_VERSION_HEX < 0x02050000)
  78 typedef int Py_ssize_t;
  79 #endif
  80
  81 /**
  82  * A cache of the spec_args for a set or list,
  83  * so we don't have to keep calling PyTuple_GET_ITEM.
  84  */
  85 typedef struct {
  86   TType element_type;
  87   PyObject* typeargs;
  88 } SetListTypeArgs;
  89
  90 /**
  91  * A cache of the spec_args for a map,
  92  * so we don't have to keep calling PyTuple_GET_ITEM.
  93  */
  94 typedef struct {
  95   TType ktag;
  96   TType vtag;
  97   PyObject* ktypeargs;
  98   PyObject* vtypeargs;
  99 } MapTypeArgs;
 100
 101 /**
 102  * A cache of the spec_args for a struct,
 103  * so we don't have to keep calling PyTuple_GET_ITEM.
 104  */
 105 typedef struct {
 106   PyObject* klass;
 107   PyObject* spec;
 108 } StructTypeArgs;
 109
 110 /**
 111  * A cache of the item spec from a struct specification,
 112  * so we don't have to keep calling PyTuple_GET_ITEM.
 113  */
 114 typedef struct {
 115   int tag;
 116   TType type;
 117   PyObject* attrname;
 118   PyObject* typeargs;
 119   PyObject* defval;
 120 } StructItemSpec;
 121
 122 /**
 123  * A cache of the two key attributes of a CReadableTransport,
 124  * so we don't have to keep calling PyObject_GetAttr.
 125  */
 126 typedef struct {
 127   PyObject* stringiobuf;
 128   PyObject* refill_callable;
 129 } DecodeBuffer;
 130
 131 /** Pointer to interned string to speed up attribute lookup. */
 132 static PyObject* INTERN_STRING(cstringio_buf);
 133 /** Pointer to interned string to speed up attribute lookup. */
 134 static PyObject* INTERN_STRING(cstringio_refill);
 135
 136 static inline bool
 137 check_ssize_t_32(Py_ssize_t len) {
 138   // error from getting the int
 139   if (INT_CONV_ERROR_OCCURRED(len)) {
 140     return false;
 141   }
 142   if (!CHECK_RANGE(len, 0, INT32_MAX)) {
 143     PyErr_SetString(PyExc_OverflowError, "string size out of range");
 144     return false;
 145   }
 146   return true;
 147 }
 148
 149 static inline bool
 150 parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) {
 151   long val = PyInt_AsLong(o);
 152
 153   if (INT_CONV_ERROR_OCCURRED(val)) {
 154     return false;
 155   }
 156   if (!CHECK_RANGE(val, min, max)) {
 157     PyErr_SetString(PyExc_OverflowError, "int out of range");
 158     return false;
 159   }
 160
 161   *ret = (int32_t) val;
 162   return true;
 163 }
 164
 165
 166 /* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */
 167
 168 static bool
 169 parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) {
 170   if (PyTuple_Size(typeargs) != 2) {
 171     PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args");
 172     return false;
 173   }
 174
 175   dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
 176   if (INT_CONV_ERROR_OCCURRED(dest->element_type)) {
 177     return false;
 178   }
 179
 180   dest->typeargs = PyTuple_GET_ITEM(typeargs, 1);
 181
 182   return true;
 183 }
 184
 185 static bool
 186 parse_map_args(MapTypeArgs* dest, PyObject* typeargs) {
 187   if (PyTuple_Size(typeargs) != 4) {
 188     PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map");
 189     return false;
 190   }
 191
 192   dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
 193   if (INT_CONV_ERROR_OCCURRED(dest->ktag)) {
 194     return false;
 195   }
 196
 197   dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2));
 198   if (INT_CONV_ERROR_OCCURRED(dest->vtag)) {
 199     return false;
 200   }
 201
 202   dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1);
 203   dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3);
 204
 205   return true;
 206 }
 207
 208 static bool
 209 parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) {
 210   if (PyTuple_Size(typeargs) != 2) {
 211     PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args");
 212     return false;
 213   }
 214
 215   dest->klass = PyTuple_GET_ITEM(typeargs, 0);
 216   dest->spec = PyTuple_GET_ITEM(typeargs, 1);
 217
 218   return true;
 219 }
 220
 221 static int
 222 parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
 223
 224   // i'd like to use ParseArgs here, but it seems to be a bottleneck.
 225   if (PyTuple_Size(spec_tuple) != 5) {
 226     PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple");
 227     return false;
 228   }
 229
 230   dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0));
 231   if (INT_CONV_ERROR_OCCURRED(dest->tag)) {
 232     return false;
 233   }
 234
 235   dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1));
 236   if (INT_CONV_ERROR_OCCURRED(dest->type)) {
 237     return false;
 238   }
 239
 240   dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2);
 241   dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3);
 242   dest->defval = PyTuple_GET_ITEM(spec_tuple, 4);
 243   return true;
 244 }
 245
 246 /* ====== END UTILITIES ====== */
 247
 248
 249 /* ====== BEGIN WRITING FUNCTIONS ====== */
 250
 251 /* --- LOW-LEVEL WRITING FUNCTIONS --- */
 252
 253 static void writeByte(PyObject* outbuf, int8_t val) {
 254   int8_t net = val;
 255   PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t));
 256 }
 257
 258 static void writeI16(PyObject* outbuf, int16_t val) {
 259   int16_t net = (int16_t)htons(val);
 260   PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t));
 261 }
 262
 263 static void writeI32(PyObject* outbuf, int32_t val) {
 264   int32_t net = (int32_t)htonl(val);
 265   PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t));
 266 }
 267
 268 static void writeI64(PyObject* outbuf, int64_t val) {
 269   int64_t net = (int64_t)htonll(val);
 270   PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t));
 271 }
 272
 273 static void writeDouble(PyObject* outbuf, double dub) {
 274   // Unfortunately, bitwise_cast doesn't work in C.  Bad C!
 275   union {
 276     double f;
 277     int64_t t;
 278   } transfer;
 279   transfer.f = dub;
 280   writeI64(outbuf, transfer.t);
 281 }
 282
 283
 284 /* --- MAIN RECURSIVE OUTPUT FUCNTION -- */
 285
 286 static int
 287 output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) {
 288   /*
 289    * Refcounting Strategy:
 290    *
 291    * We assume that elements of the thrift_spec tuple are not going to be
 292    * mutated, so we don't ref count those at all. Other than that, we try to
 293    * keep a reference to all the user-created objects while we work with them.
 294    * output_val assumes that a reference is already held. The *caller* is
 295    * responsible for handling references
 296    */
 297
 298   switch (type) {
 299
 300   case T_BOOL: {
 301     int v = PyObject_IsTrue(value);
 302     if (v == -1) {
 303       return false;
 304     }
 305
 306     writeByte(output, (int8_t) v);
 307     break;
 308   }
 309   case T_I08: {
 310     int32_t val;
 311
 312     if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) {
 313       return false;
 314     }
 315
 316     writeByte(output, (int8_t) val);
 317     break;
 318   }
 319   case T_I16: {
 320     int32_t val;
 321
 322     if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) {
 323       return false;
 324     }
 325
 326     writeI16(output, (int16_t) val);
 327     break;
 328   }
 329   case T_I32: {
 330     int32_t val;
 331
 332     if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) {
 333       return false;
 334     }
 335
 336     writeI32(output, val);
 337     break;
 338   }
 339   case T_I64: {
 340     int64_t nval = PyLong_AsLongLong(value);
 341
 342     if (INT_CONV_ERROR_OCCURRED(nval)) {
 343       return false;
 344     }
 345
 346     if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) {
 347       PyErr_SetString(PyExc_OverflowError, "int out of range");
 348       return false;
 349     }
 350
 351     writeI64(output, nval);
 352     break;
 353   }
 354
 355   case T_DOUBLE: {
 356     double nval = PyFloat_AsDouble(value);
 357     if (nval == -1.0 && PyErr_Occurred()) {
 358       return false;
 359     }
 360
 361     writeDouble(output, nval);
 362     break;
 363   }
 364
 365   case T_STRING: {
 366     Py_ssize_t len = PyString_Size(value);
 367
 368     if (!check_ssize_t_32(len)) {
 369       return false;
 370     }
 371
 372     writeI32(output, (int32_t) len);
 373     PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len);
 374     break;
 375   }
 376
 377   case T_LIST:
 378   case T_SET: {
 379     Py_ssize_t len;
 380     SetListTypeArgs parsedargs;
 381     PyObject *item;
 382     PyObject *iterator;
 383
 384     if (!parse_set_list_args(&parsedargs, typeargs)) {
 385       return false;
 386     }
 387
 388     len = PyObject_Length(value);
 389
 390     if (!check_ssize_t_32(len)) {
 391       return false;
 392     }
 393
 394     writeByte(output, parsedargs.element_type);
 395     writeI32(output, (int32_t) len);
 396
 397     iterator =  PyObject_GetIter(value);
 398     if (iterator == NULL) {
 399       return false;
 400     }
 401
 402     while ((item = PyIter_Next(iterator))) {
 403       if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) {
 404         Py_DECREF(item);
 405         Py_DECREF(iterator);
 406         return false;
 407       }
 408       Py_DECREF(item);
 409     }
 410
 411     Py_DECREF(iterator);
 412
 413     if (PyErr_Occurred()) {
 414       return false;
 415     }
 416
 417     break;
 418   }
 419
 420   case T_MAP: {
 421     PyObject *k, *v;
 422     int pos = 0;
 423     Py_ssize_t len;
 424
 425     MapTypeArgs parsedargs;
 426
 427     len = PyDict_Size(value);
 428     if (!check_ssize_t_32(len)) {
 429       return false;
 430     }
 431
 432     if (!parse_map_args(&parsedargs, typeargs)) {
 433       return false;
 434     }
 435
 436     writeByte(output, parsedargs.ktag);
 437     writeByte(output, parsedargs.vtag);
 438     writeI32(output, len);
 439
 440     // TODO(bmaurer): should support any mapping, not just dicts
 441     while (PyDict_Next(value, &pos, &k, &v)) {
 442       // TODO(dreiss): Think hard about whether these INCREFs actually
 443       //               turn any unsafe scenarios into safe scenarios.
 444       Py_INCREF(k);
 445       Py_INCREF(v);
 446
 447       if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs)
 448           || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) {
 449         Py_DECREF(k);
 450         Py_DECREF(v);
 451         return false;
 452       }
 453     }
 454     break;
 455   }
 456
 457   // TODO(dreiss): Consider breaking this out as a function
 458   //               the way we did for decode_struct.
 459   case T_STRUCT: {
 460     StructTypeArgs parsedargs;
 461     Py_ssize_t nspec;
 462     Py_ssize_t i;
 463
 464     if (!parse_struct_args(&parsedargs, typeargs)) {
 465       return false;
 466     }
 467
 468     nspec = PyTuple_Size(parsedargs.spec);
 469
 470     if (nspec == -1) {
 471       return false;
 472     }
 473
 474     for (i = 0; i < nspec; i++) {
 475       StructItemSpec parsedspec;
 476       PyObject* spec_tuple;
 477       PyObject* instval = NULL;
 478
 479       spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i);
 480       if (spec_tuple == Py_None) {
 481         continue;
 482       }
 483
 484       if (!parse_struct_item_spec (&parsedspec, spec_tuple)) {
 485         return false;
 486       }
 487
 488       instval = PyObject_GetAttr(value, parsedspec.attrname);
 489
 490       if (!instval) {
 491         return false;
 492       }
 493
 494       if (instval == Py_None) {
 495         Py_DECREF(instval);
 496         continue;
 497       }
 498
 499       writeByte(output, (int8_t) parsedspec.type);
 500       writeI16(output, parsedspec.tag);
 501
 502       if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) {
 503         Py_DECREF(instval);
 504         return false;
 505       }
 506
 507       Py_DECREF(instval);
 508     }
 509
 510     writeByte(output, (int8_t)T_STOP);
 511     break;
 512   }
 513
 514   case T_STOP:
 515   case T_VOID:
 516   case T_UTF16:
 517   case T_UTF8:
 518   case T_U64:
 519   default:
 520     PyErr_SetString(PyExc_TypeError, "Unexpected TType");
 521     return false;
 522
 523   }
 524
 525   return true;
 526 }
 527
 528
 529 /* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */
 530
 531 static PyObject *
 532 encode_binary(PyObject *self, PyObject *args) {
 533   PyObject* enc_obj;
 534   PyObject* type_args;
 535   PyObject* buf;
 536   PyObject* ret = NULL;
 537
 538   if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) {
 539     return NULL;
 540   }
 541
 542   buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE);
 543   if (output_val(buf, enc_obj, T_STRUCT, type_args)) {
 544     ret = PycStringIO->cgetvalue(buf);
 545   }
 546
 547   Py_DECREF(buf);
 548   return ret;
 549 }
 550
 551 /* ====== END WRITING FUNCTIONS ====== */
 552
 553
 554 /* ====== BEGIN READING FUNCTIONS ====== */
 555
 556 /* --- LOW-LEVEL READING FUNCTIONS --- */
 557
 558 static void
 559 free_decodebuf(DecodeBuffer* d) {
 560   Py_XDECREF(d->stringiobuf);
 561   Py_XDECREF(d->refill_callable);
 562 }
 563
 564 static bool
 565 decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) {
 566   dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf));
 567   if (!dest->stringiobuf) {
 568     return false;
 569   }
 570
 571   if (!PycStringIO_InputCheck(dest->stringiobuf)) {
 572     free_decodebuf(dest);
 573     PyErr_SetString(PyExc_TypeError, "expecting stringio input");
 574     return false;
 575   }
 576
 577   dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill));
 578
 579   if(!dest->refill_callable) {
 580     free_decodebuf(dest);
 581     return false;
 582   }
 583
 584   if (!PyCallable_Check(dest->refill_callable)) {
 585     free_decodebuf(dest);
 586     PyErr_SetString(PyExc_TypeError, "expecting callable");
 587     return false;
 588   }
 589
 590   return true;
 591 }
 592
 593 static bool readBytes(DecodeBuffer* input, char** output, int len) {
 594   int read;
 595
 596   // TODO(dreiss): Don't fear the malloc.  Think about taking a copy of
 597   //               the partial read instead of forcing the transport
 598   //               to prepend it to its buffer.
 599
 600   read = PycStringIO->cread(input->stringiobuf, output, len);
 601
 602   if (read == len) {
 603     return true;
 604   } else if (read == -1) {
 605     return false;
 606   } else {
 607     PyObject* newiobuf;
 608
 609     // using building functions as this is a rare codepath
 610     newiobuf = PyObject_CallFunction(
 611         input->refill_callable, "s#i", *output, read, len, NULL);
 612     if (newiobuf == NULL) {
 613       return false;
 614     }
 615
 616     // must do this *AFTER* the call so that we don't deref the io buffer
 617     Py_CLEAR(input->stringiobuf);
 618     input->stringiobuf = newiobuf;
 619
 620     read = PycStringIO->cread(input->stringiobuf, output, len);
 621
 622     if (read == len) {
 623       return true;
 624     } else if (read == -1) {
 625       return false;
 626     } else {
 627       // TODO(dreiss): This could be a valid code path for big binary blobs.
 628       PyErr_SetString(PyExc_TypeError,
 629           "refill claimed to have refilled the buffer, but didn't!!");
 630       return false;
 631     }
 632   }
 633 }
 634
 635 static int8_t readByte(DecodeBuffer* input) {
 636   char* buf;
 637   if (!readBytes(input, &buf, sizeof(int8_t))) {
 638     return -1;
 639   }
 640
 641   return *(int8_t*) buf;
 642 }
 643
 644 static int16_t readI16(DecodeBuffer* input) {
 645   char* buf;
 646   if (!readBytes(input, &buf, sizeof(int16_t))) {
 647     return -1;
 648   }
 649
 650   return (int16_t) ntohs(*(int16_t*) buf);
 651 }
 652
 653 static int32_t readI32(DecodeBuffer* input) {
 654   char* buf;
 655   if (!readBytes(input, &buf, sizeof(int32_t))) {
 656     return -1;
 657   }
 658   return (int32_t) ntohl(*(int32_t*) buf);
 659 }
 660
 661
 662 static int64_t readI64(DecodeBuffer* input) {
 663   char* buf;
 664   if (!readBytes(input, &buf, sizeof(int64_t))) {
 665     return -1;
 666   }
 667
 668   return (int64_t) ntohll(*(int64_t*) buf);
 669 }
 670
 671 static double readDouble(DecodeBuffer* input) {
 672   union {
 673     int64_t f;
 674     double t;
 675   } transfer;
 676
 677   transfer.f = readI64(input);
 678   if (transfer.f == -1) {
 679     return -1;
 680   }
 681   return transfer.t;
 682 }
 683
 684 static bool
 685 checkTypeByte(DecodeBuffer* input, TType expected) {
 686   TType got = readByte(input);
 687   if (INT_CONV_ERROR_OCCURRED(got)) {
 688     return false;
 689   }
 690
 691   if (expected != got) {
 692     PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field");
 693     return false;
 694   }
 695   return true;
 696 }
 697
 698 static bool
 699 skip(DecodeBuffer* input, TType type) {
 700 #define SKIPBYTES(n) \
 701   do { \
 702     if (!readBytes(input, &dummy_buf, (n))) { \
 703       return false; \
 704     } \
 705   } while(0)
 706
 707   char* dummy_buf;
 708
 709   switch (type) {
 710
 711   case T_BOOL:
 712   case T_I08: SKIPBYTES(1); break;
 713   case T_I16: SKIPBYTES(2); break;
 714   case T_I32: SKIPBYTES(4); break;
 715   case T_I64:
 716   case T_DOUBLE: SKIPBYTES(8); break;
 717
 718   case T_STRING: {
 719     // TODO(dreiss): Find out if these check_ssize_t32s are really necessary.
 720     int len = readI32(input);
 721     if (!check_ssize_t_32(len)) {
 722       return false;
 723     }
 724     SKIPBYTES(len);
 725     break;
 726   }
 727
 728   case T_LIST:
 729   case T_SET: {
 730     TType etype;
 731     int len, i;
 732
 733     etype = readByte(input);
 734     if (etype == -1) {
 735       return false;
 736     }
 737
 738     len = readI32(input);
 739     if (!check_ssize_t_32(len)) {
 740       return false;
 741     }
 742
 743     for (i = 0; i < len; i++) {
 744       if (!skip(input, etype)) {
 745         return false;
 746       }
 747     }
 748     break;
 749   }
 750
 751   case T_MAP: {
 752     TType ktype, vtype;
 753     int len, i;
 754
 755     ktype = readByte(input);
 756     if (ktype == -1) {
 757       return false;
 758     }
 759
 760     vtype = readByte(input);
 761     if (vtype == -1) {
 762       return false;
 763     }
 764
 765     len = readI32(input);
 766     if (!check_ssize_t_32(len)) {
 767       return false;
 768     }
 769
 770     for (i = 0; i < len; i++) {
 771       if (!(skip(input, ktype) && skip(input, vtype))) {
 772         return false;
 773       }
 774     }
 775     break;
 776   }
 777
 778   case T_STRUCT: {
 779     while (true) {
 780       TType type;
 781
 782       type = readByte(input);
 783       if (type == -1) {
 784         return false;
 785       }
 786
 787       if (type == T_STOP)
 788         break;
 789
 790       SKIPBYTES(2); // tag
 791       if (!skip(input, type)) {
 792         return false;
 793       }
 794     }
 795     break;
 796   }
 797
 798   case T_STOP:
 799   case T_VOID:
 800   case T_UTF16:
 801   case T_UTF8:
 802   case T_U64:
 803   default:
 804     PyErr_SetString(PyExc_TypeError, "Unexpected TType");
 805     return false;
 806
 807   }
 808
 809   return false;
 810
 811 #undef SKIPBYTES
 812 }
 813
 814
 815 /* --- HELPER FUNCTION FOR DECODE_VAL --- */
 816
 817 static PyObject*
 818 decode_val(DecodeBuffer* input, TType type, PyObject* typeargs);
 819
 820 static bool
 821 decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) {
 822   int spec_seq_len = PyTuple_Size(spec_seq);
 823   if (spec_seq_len == -1) {
 824     return false;
 825   }
 826
 827   while (true) {
 828     TType type;
 829     int16_t tag;
 830     PyObject* item_spec;
 831     PyObject* fieldval = NULL;
 832     StructItemSpec parsedspec;
 833
 834     type = readByte(input);
 835     if (type == -1) {
 836       return false;
 837     }
 838     if (type == T_STOP) {
 839       break;
 840     }
 841     tag = readI16(input);
 842     if (INT_CONV_ERROR_OCCURRED(tag)) {
 843       return false;
 844     }
 845     if (tag >= 0 && tag < spec_seq_len) {
 846       item_spec = PyTuple_GET_ITEM(spec_seq, tag);
 847     } else {
 848       item_spec = Py_None;
 849     }
 850
 851     if (item_spec == Py_None) {
 852       if (!skip(input, type)) {
 853         return false;
 854       }
 855     }
 856
 857     if (!parse_struct_item_spec(&parsedspec, item_spec)) {
 858       return false;
 859     }
 860     if (parsedspec.type != type) {
 861       PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading");
 862       return false;
 863     }
 864
 865     fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs);
 866     if (fieldval == NULL) {
 867       return false;
 868     }
 869
 870     if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) {
 871       Py_DECREF(fieldval);
 872       return false;
 873     }
 874     Py_DECREF(fieldval);
 875   }
 876   return true;
 877 }
 878
 879
 880 /* --- MAIN RECURSIVE INPUT FUCNTION --- */
 881
 882 // Returns a new reference.
 883 static PyObject*
 884 decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
 885   switch (type) {
 886
 887   case T_BOOL: {
 888     int8_t v = readByte(input);
 889     if (INT_CONV_ERROR_OCCURRED(v)) {
 890       return NULL;
 891     }
 892
 893     switch (v) {
 894     case 0: Py_RETURN_FALSE;
 895     case 1: Py_RETURN_TRUE;
 896     // Don't laugh.  This is a potentially serious issue.
 897     default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL;
 898     }
 899     break;
 900   }
 901   case T_I08: {
 902     int8_t v = readByte(input);
 903     if (INT_CONV_ERROR_OCCURRED(v)) {
 904       return NULL;
 905     }
 906
 907     return PyInt_FromLong(v);
 908   }
 909   case T_I16: {
 910     int16_t v = readI16(input);
 911     if (INT_CONV_ERROR_OCCURRED(v)) {
 912       return NULL;
 913     }
 914     return PyInt_FromLong(v);
 915   }
 916   case T_I32: {
 917     int32_t v = readI32(input);
 918     if (INT_CONV_ERROR_OCCURRED(v)) {
 919       return NULL;
 920     }
 921     return PyInt_FromLong(v);
 922   }
 923
 924   case T_I64: {
 925     int64_t v = readI64(input);
 926     if (INT_CONV_ERROR_OCCURRED(v)) {
 927       return NULL;
 928     }
 929     // TODO(dreiss): Find out if we can take this fastpath always when
 930     //               sizeof(long) == sizeof(long long).
 931     if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) {
 932       return PyInt_FromLong((long) v);
 933     }
 934
 935     return PyLong_FromLongLong(v);
 936   }
 937
 938   case T_DOUBLE: {
 939     double v = readDouble(input);
 940     if (v == -1.0 && PyErr_Occurred()) {
 941       return false;
 942     }
 943     return PyFloat_FromDouble(v);
 944   }
 945
 946   case T_STRING: {
 947     Py_ssize_t len = readI32(input);
 948     char* buf;
 949     if (!readBytes(input, &buf, len)) {
 950       return NULL;
 951     }
 952
 953     return PyString_FromStringAndSize(buf, len);
 954   }
 955
 956   case T_LIST:
 957   case T_SET: {
 958     SetListTypeArgs parsedargs;
 959     int32_t len;
 960     PyObject* ret = NULL;
 961     int i;
 962
 963     if (!parse_set_list_args(&parsedargs, typeargs)) {
 964       return NULL;
 965     }
 966
 967     if (!checkTypeByte(input, parsedargs.element_type)) {
 968       return NULL;
 969     }
 970
 971     len = readI32(input);
 972     if (!check_ssize_t_32(len)) {
 973       return NULL;
 974     }
 975
 976     ret = PyList_New(len);
 977     if (!ret) {
 978       return NULL;
 979     }
 980
 981     for (i = 0; i < len; i++) {
 982       PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs);
 983       if (!item) {
 984         Py_DECREF(ret);
 985         return NULL;
 986       }
 987       PyList_SET_ITEM(ret, i, item);
 988     }
 989
 990     // TODO(dreiss): Consider biting the bullet and making two separate cases
 991     //               for list and set, avoiding this post facto conversion.
 992     if (type == T_SET) {
 993       PyObject* setret;
 994 #if (PY_VERSION_HEX < 0x02050000)
 995       // hack needed for older versions
 996       setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL);
 997 #else
 998       // official version
 999       setret = PySet_New(ret);
1000 #endif
1001       Py_DECREF(ret);
1002       return setret;
1003     }
1004     return ret;
1005   }
1006
1007   case T_MAP: {
1008     int32_t len;
1009     int i;
1010     MapTypeArgs parsedargs;
1011     PyObject* ret = NULL;
1012
1013     if (!parse_map_args(&parsedargs, typeargs)) {
1014       return NULL;
1015     }
1016
1017     if (!checkTypeByte(input, parsedargs.ktag)) {
1018       return NULL;
1019     }
1020     if (!checkTypeByte(input, parsedargs.vtag)) {
1021       return NULL;
1022     }
1023
1024     len = readI32(input);
1025     if (!check_ssize_t_32(len)) {
1026       return false;
1027     }
1028
1029     ret = PyDict_New();
1030     if (!ret) {
1031       goto error;
1032     }
1033
1034     for (i = 0; i < len; i++) {
1035       PyObject* k = NULL;
1036       PyObject* v = NULL;
1037       k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs);
1038       if (k == NULL) {
1039         goto loop_error;
1040       }
1041       v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs);
1042       if (v == NULL) {
1043         goto loop_error;
1044       }
1045       if (PyDict_SetItem(ret, k, v) == -1) {
1046         goto loop_error;
1047       }
1048
1049       Py_DECREF(k);
1050       Py_DECREF(v);
1051       continue;
1052
1053       // Yuck!  Destructors, anyone?
1054       loop_error:
1055       Py_XDECREF(k);
1056       Py_XDECREF(v);
1057       goto error;
1058     }
1059
1060     return ret;
1061
1062     error:
1063     Py_XDECREF(ret);
1064     return NULL;
1065   }
1066
1067   case T_STRUCT: {
1068     StructTypeArgs parsedargs;
1069     if (!parse_struct_args(&parsedargs, typeargs)) {
1070       return NULL;
1071     }
1072
1073     PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL);
1074     if (!ret) {
1075       return NULL;
1076     }
1077
1078     if (!decode_struct(input, ret, parsedargs.spec)) {
1079       Py_DECREF(ret);
1080       return NULL;
1081     }
1082
1083     return ret;
1084   }
1085
1086   case T_STOP:
1087   case T_VOID:
1088   case T_UTF16:
1089   case T_UTF8:
1090   case T_U64:
1091   default:
1092     PyErr_SetString(PyExc_TypeError, "Unexpected TType");
1093     return NULL;
1094   }
1095 }
1096
1097
1098 /* --- TOP-LEVEL WRAPPER FOR INPUT -- */
1099
1100 static PyObject*
1101 decode_binary(PyObject *self, PyObject *args) {
1102   PyObject* output_obj = NULL;
1103   PyObject* transport = NULL;
1104   PyObject* typeargs = NULL;
1105   StructTypeArgs parsedargs;
1106   DecodeBuffer input = {};
1107
1108   if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
1109     return NULL;
1110   }
1111
1112   if (!parse_struct_args(&parsedargs, typeargs)) {
1113     return NULL;
1114   }
1115
1116   if (!decode_buffer_from_obj(&input, transport)) {
1117     return NULL;
1118   }
1119
1120   if (!decode_struct(&input, output_obj, parsedargs.spec)) {
1121     free_decodebuf(&input);
1122     return NULL;
1123   }
1124
1125   free_decodebuf(&input);
1126
1127   Py_RETURN_NONE;
1128 }
1129
1130 /* ====== END READING FUNCTIONS ====== */
1131
1132
1133 /* -- PYTHON MODULE SETUP STUFF --- */
1134
1135 static PyMethodDef ThriftFastBinaryMethods[] = {
1136
1137   {"encode_binary",  encode_binary, METH_VARARGS, ""},
1138   {"decode_binary",  decode_binary, METH_VARARGS, ""},
1139
1140   {NULL, NULL, 0, NULL}        /* Sentinel */
1141 };
1142
1143 PyMODINIT_FUNC
1144 initfastbinary(void) {
1145 #define INIT_INTERN_STRING(value) \
1146   do { \
1147     INTERN_STRING(value) = PyString_InternFromString(#value); \
1148     if(!INTERN_STRING(value)) return; \
1149   } while(0)
1150
1151   INIT_INTERN_STRING(cstringio_buf);
1152   INIT_INTERN_STRING(cstringio_refill);
1153 #undef INIT_INTERN_STRING
1154
1155   PycString_IMPORT;
1156   if (PycStringIO == NULL) return;
1157
1158   (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
1159 }