Python/marshal.c

   1
   2 /* Write Python objects to files and read them back.
   3    This is intended for writing and reading compiled Python code only;
   4    a true persistent storage facility would be much harder, since
   5    it would have to take circular links and sharing into account. */
   6
   7 #define PY_SSIZE_T_CLEAN
   8
   9 #include "Python.h"
  10 #include "longintrepr.h"
  11 #include "code.h"
  12 #include "marshal.h"
  13
  14 /* High water mark to determine when the marshalled object is dangerously deep
  15  * and risks coring the interpreter.  When the object stack gets this deep,
  16  * raise an exception instead of continuing.
  17  */
  18 #define MAX_MARSHAL_STACK_DEPTH 5000
  19
  20 #define TYPE_NULL               '0'
  21 #define TYPE_NONE               'N'
  22 #define TYPE_FALSE              'F'
  23 #define TYPE_TRUE               'T'
  24 #define TYPE_STOPITER           'S'
  25 #define TYPE_ELLIPSIS           '.'
  26 #define TYPE_INT                'i'
  27 #define TYPE_INT64              'I'
  28 #define TYPE_FLOAT              'f'
  29 #define TYPE_BINARY_FLOAT       'g'
  30 #define TYPE_COMPLEX            'x'
  31 #define TYPE_BINARY_COMPLEX     'y'
  32 #define TYPE_LONG               'l'
  33 #define TYPE_STRING             's'
  34 #define TYPE_INTERNED           't'
  35 #define TYPE_STRINGREF          'R'
  36 #define TYPE_TUPLE              '('
  37 #define TYPE_LIST               '['
  38 #define TYPE_DICT               '{'
  39 #define TYPE_CODE               'c'
  40 #define TYPE_UNICODE            'u'
  41 #define TYPE_UNKNOWN            '?'
  42 #define TYPE_SET                '<'
  43 #define TYPE_FROZENSET          '>'
  44
  45 typedef struct {
  46         FILE *fp;
  47         int error;
  48         int depth;
  49         /* If fp == NULL, the following are valid: */
  50         PyObject *str;
  51         char *ptr;
  52         char *end;
  53         PyObject *strings; /* dict on marshal, list on unmarshal */
  54         int version;
  55 } WFILE;
  56
  57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  58                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  59                            else w_more(c, p)
  60
  61 static void
  62 w_more(int c, WFILE *p)
  63 {
  64         Py_ssize_t size, newsize;
  65         if (p->str == NULL)
  66                 return; /* An error already occurred */
  67         size = PyString_Size(p->str);
  68         newsize = size + 1024;
  69         if (_PyString_Resize(&p->str, newsize) != 0) {
  70                 p->ptr = p->end = NULL;
  71         }
  72         else {
  73                 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
  74                 p->end =
  75                         PyString_AS_STRING((PyStringObject *)p->str) + newsize;
  76                 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
  77         }
  78 }
  79
  80 static void
  81 w_string(char *s, int n, WFILE *p)
  82 {
  83         if (p->fp != NULL) {
  84                 fwrite(s, 1, n, p->fp);
  85         }
  86         else {
  87                 while (--n >= 0) {
  88                         w_byte(*s, p);
  89                         s++;
  90                 }
  91         }
  92 }
  93
  94 static void
  95 w_short(int x, WFILE *p)
  96 {
  97         w_byte((char)( x      & 0xff), p);
  98         w_byte((char)((x>> 8) & 0xff), p);
  99 }
 100
 101 static void
 102 w_long(long x, WFILE *p)
 103 {
 104         w_byte((char)( x      & 0xff), p);
 105         w_byte((char)((x>> 8) & 0xff), p);
 106         w_byte((char)((x>>16) & 0xff), p);
 107         w_byte((char)((x>>24) & 0xff), p);
 108 }
 109
 110 #if SIZEOF_LONG > 4
 111 static void
 112 w_long64(long x, WFILE *p)
 113 {
 114         w_long(x, p);
 115         w_long(x>>32, p);
 116 }
 117 #endif
 118
 119 static void
 120 w_object(PyObject *v, WFILE *p)
 121 {
 122         Py_ssize_t i, n;
 123
 124         p->depth++;
 125
 126         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 127                 p->error = 2;
 128         }
 129         else if (v == NULL) {
 130                 w_byte(TYPE_NULL, p);
 131         }
 132         else if (v == Py_None) {
 133                 w_byte(TYPE_NONE, p);
 134         }
 135         else if (v == PyExc_StopIteration) {
 136                 w_byte(TYPE_STOPITER, p);
 137         }
 138         else if (v == Py_Ellipsis) {
 139                 w_byte(TYPE_ELLIPSIS, p);
 140         }
 141         else if (v == Py_False) {
 142                 w_byte(TYPE_FALSE, p);
 143         }
 144         else if (v == Py_True) {
 145                 w_byte(TYPE_TRUE, p);
 146         }
 147         else if (PyInt_Check(v)) {
 148                 long x = PyInt_AS_LONG((PyIntObject *)v);
 149 #if SIZEOF_LONG > 4
 150                 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
 151                 if (y && y != -1) {
 152                         w_byte(TYPE_INT64, p);
 153                         w_long64(x, p);
 154                 }
 155                 else
 156 #endif
 157                         {
 158                         w_byte(TYPE_INT, p);
 159                         w_long(x, p);
 160                 }
 161         }
 162         else if (PyLong_Check(v)) {
 163                 PyLongObject *ob = (PyLongObject *)v;
 164                 w_byte(TYPE_LONG, p);
 165                 n = ob->ob_size;
 166                 w_long((long)n, p);
 167                 if (n < 0)
 168                         n = -n;
 169                 for (i = 0; i < n; i++)
 170                         w_short(ob->ob_digit[i], p);
 171         }
 172         else if (PyFloat_Check(v)) {
 173                 if (p->version > 1) {
 174                         unsigned char buf[8];
 175                         if (_PyFloat_Pack8(PyFloat_AsDouble(v),
 176                                            buf, 1) < 0) {
 177                                 p->error = 1;
 178                                 return;
 179                         }
 180                         w_byte(TYPE_BINARY_FLOAT, p);
 181                         w_string((char*)buf, 8, p);
 182                 }
 183                 else {
 184                         char buf[256]; /* Plenty to format any double */
 185                         PyFloat_AsReprString(buf, (PyFloatObject *)v);
 186                         n = strlen(buf);
 187                         w_byte(TYPE_FLOAT, p);
 188                         w_byte((int)n, p);
 189                         w_string(buf, n, p);
 190                 }
 191         }
 192 #ifndef WITHOUT_COMPLEX
 193         else if (PyComplex_Check(v)) {
 194                 if (p->version > 1) {
 195                         unsigned char buf[8];
 196                         if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
 197                                            buf, 1) < 0) {
 198                                 p->error = 1;
 199                                 return;
 200                         }
 201                         w_byte(TYPE_BINARY_COMPLEX, p);
 202                         w_string((char*)buf, 8, p);
 203                         if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
 204                                            buf, 1) < 0) {
 205                                 p->error = 1;
 206                                 return;
 207                         }
 208                         w_string((char*)buf, 8, p);
 209                 }
 210                 else {
 211                         char buf[256]; /* Plenty to format any double */
 212                         PyFloatObject *temp;
 213                         w_byte(TYPE_COMPLEX, p);
 214                         temp = (PyFloatObject*)PyFloat_FromDouble(
 215                                 PyComplex_RealAsDouble(v));
 216                         PyFloat_AsReprString(buf, temp);
 217                         Py_DECREF(temp);
 218                         n = (int)strlen(buf);
 219                         w_byte(n, p);
 220                         w_string(buf, n, p);
 221                         temp = (PyFloatObject*)PyFloat_FromDouble(
 222                                 PyComplex_ImagAsDouble(v));
 223                         PyFloat_AsReprString(buf, temp);
 224                         Py_DECREF(temp);
 225                         n = (int)strlen(buf);
 226                         w_byte(n, p);
 227                         w_string(buf, n, p);
 228                 }
 229         }
 230 #endif
 231         else if (PyString_Check(v)) {
 232                 if (p->strings && PyString_CHECK_INTERNED(v)) {
 233                         PyObject *o = PyDict_GetItem(p->strings, v);
 234                         if (o) {
 235                                 long w = PyInt_AsLong(o);
 236                                 w_byte(TYPE_STRINGREF, p);
 237                                 w_long(w, p);
 238                                 goto exit;
 239                         }
 240                         else {
 241                                 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
 242                                 PyDict_SetItem(p->strings, v, o);
 243                                 Py_DECREF(o);
 244                                 w_byte(TYPE_INTERNED, p);
 245                         }
 246                 }
 247                 else {
 248                         w_byte(TYPE_STRING, p);
 249                 }
 250                 n = PyString_GET_SIZE(v);
 251                 w_long((long)n, p);
 252                 w_string(PyString_AS_STRING(v), n, p);
 253         }
 254 #ifdef Py_USING_UNICODE
 255         else if (PyUnicode_Check(v)) {
 256                 PyObject *utf8;
 257                 utf8 = PyUnicode_AsUTF8String(v);
 258                 if (utf8 == NULL) {
 259                         p->depth--;
 260                         p->error = 1;
 261                         return;
 262                 }
 263                 w_byte(TYPE_UNICODE, p);
 264                 n = PyString_GET_SIZE(utf8);
 265                 w_long((long)n, p);
 266                 w_string(PyString_AS_STRING(utf8), n, p);
 267                 Py_DECREF(utf8);
 268         }
 269 #endif
 270         else if (PyTuple_Check(v)) {
 271                 w_byte(TYPE_TUPLE, p);
 272                 n = PyTuple_Size(v);
 273                 w_long((long)n, p);
 274                 for (i = 0; i < n; i++) {
 275                         w_object(PyTuple_GET_ITEM(v, i), p);
 276                 }
 277         }
 278         else if (PyList_Check(v)) {
 279                 w_byte(TYPE_LIST, p);
 280                 n = PyList_GET_SIZE(v);
 281                 w_long((long)n, p);
 282                 for (i = 0; i < n; i++) {
 283                         w_object(PyList_GET_ITEM(v, i), p);
 284                 }
 285         }
 286         else if (PyDict_Check(v)) {
 287                 Py_ssize_t pos;
 288                 PyObject *key, *value;
 289                 w_byte(TYPE_DICT, p);
 290                 /* This one is NULL object terminated! */
 291                 pos = 0;
 292                 while (PyDict_Next(v, &pos, &key, &value)) {
 293                         w_object(key, p);
 294                         w_object(value, p);
 295                 }
 296                 w_object((PyObject *)NULL, p);
 297         }
 298         else if (PyAnySet_Check(v)) {
 299                 PyObject *value, *it;
 300
 301                 if (PyObject_TypeCheck(v, &PySet_Type))
 302                         w_byte(TYPE_SET, p);
 303                 else
 304                         w_byte(TYPE_FROZENSET, p);
 305                 n = PyObject_Size(v);
 306                 if (n == -1) {
 307                         p->depth--;
 308                         p->error = 1;
 309                         return;
 310                 }
 311                 w_long((long)n, p);
 312                 it = PyObject_GetIter(v);
 313                 if (it == NULL) {
 314                         p->depth--;
 315                         p->error = 1;
 316                         return;
 317                 }
 318                 while ((value = PyIter_Next(it)) != NULL) {
 319                         w_object(value, p);
 320                         Py_DECREF(value);
 321                 }
 322                 Py_DECREF(it);
 323                 if (PyErr_Occurred()) {
 324                         p->depth--;
 325                         p->error = 1;
 326                         return;
 327                 }
 328         }
 329         else if (PyCode_Check(v)) {
 330                 PyCodeObject *co = (PyCodeObject *)v;
 331                 w_byte(TYPE_CODE, p);
 332                 w_long(co->co_argcount, p);
 333                 w_long(co->co_nlocals, p);
 334                 w_long(co->co_stacksize, p);
 335                 w_long(co->co_flags, p);
 336                 w_object(co->co_code, p);
 337                 w_object(co->co_consts, p);
 338                 w_object(co->co_names, p);
 339                 w_object(co->co_varnames, p);
 340                 w_object(co->co_freevars, p);
 341                 w_object(co->co_cellvars, p);
 342                 w_object(co->co_filename, p);
 343                 w_object(co->co_name, p);
 344                 w_long(co->co_firstlineno, p);
 345                 w_object(co->co_lnotab, p);
 346         }
 347         else if (PyObject_CheckReadBuffer(v)) {
 348                 /* Write unknown buffer-style objects as a string */
 349                 char *s;
 350                 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
 351                 w_byte(TYPE_STRING, p);
 352                 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
 353                 w_long((long)n, p);
 354                 w_string(s, n, p);
 355         }
 356         else {
 357                 w_byte(TYPE_UNKNOWN, p);
 358                 p->error = 1;
 359         }
 360    exit:
 361         p->depth--;
 362 }
 363
 364 /* version currently has no effect for writing longs. */
 365 void
 366 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 367 {
 368         WFILE wf;
 369         wf.fp = fp;
 370         wf.error = 0;
 371         wf.depth = 0;
 372         wf.strings = NULL;
 373         wf.version = version;
 374         w_long(x, &wf);
 375 }
 376
 377 void
 378 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 379 {
 380         WFILE wf;
 381         wf.fp = fp;
 382         wf.error = 0;
 383         wf.depth = 0;
 384         wf.strings = (version > 0) ? PyDict_New() : NULL;
 385         wf.version = version;
 386         w_object(x, &wf);
 387         Py_XDECREF(wf.strings);
 388 }
 389
 390 typedef WFILE RFILE; /* Same struct with different invariants */
 391
 392 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 393
 394 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 395
 396 static int
 397 r_string(char *s, int n, RFILE *p)
 398 {
 399         if (p->fp != NULL)
 400                 /* The result fits into int because it must be <=n. */
 401                 return (int)fread(s, 1, n, p->fp);
 402         if (p->end - p->ptr < n)
 403                 n = (int)(p->end - p->ptr);
 404         memcpy(s, p->ptr, n);
 405         p->ptr += n;
 406         return n;
 407 }
 408
 409 static int
 410 r_short(RFILE *p)
 411 {
 412         register short x;
 413         x = r_byte(p);
 414         x |= r_byte(p) << 8;
 415         /* Sign-extension, in case short greater than 16 bits */
 416         x |= -(x & 0x8000);
 417         return x;
 418 }
 419
 420 static long
 421 r_long(RFILE *p)
 422 {
 423         register long x;
 424         register FILE *fp = p->fp;
 425         if (fp) {
 426                 x = getc(fp);
 427                 x |= (long)getc(fp) << 8;
 428                 x |= (long)getc(fp) << 16;
 429                 x |= (long)getc(fp) << 24;
 430         }
 431         else {
 432                 x = rs_byte(p);
 433                 x |= (long)rs_byte(p) << 8;
 434                 x |= (long)rs_byte(p) << 16;
 435                 x |= (long)rs_byte(p) << 24;
 436         }
 437 #if SIZEOF_LONG > 4
 438         /* Sign extension for 64-bit machines */
 439         x |= -(x & 0x80000000L);
 440 #endif
 441         return x;
 442 }
 443
 444 /* r_long64 deals with the TYPE_INT64 code.  On a machine with
 445    sizeof(long) > 4, it returns a Python int object, else a Python long
 446    object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
 447    so there's no inefficiency here in returning a PyLong on 32-bit boxes
 448    for everything written via TYPE_INT64 (i.e., if an int is written via
 449    TYPE_INT64, it *needs* more than 32 bits).
 450 */
 451 static PyObject *
 452 r_long64(RFILE *p)
 453 {
 454         long lo4 = r_long(p);
 455         long hi4 = r_long(p);
 456 #if SIZEOF_LONG > 4
 457         long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
 458         return PyInt_FromLong(x);
 459 #else
 460         unsigned char buf[8];
 461         int one = 1;
 462         int is_little_endian = (int)*(char*)&one;
 463         if (is_little_endian) {
 464                 memcpy(buf, &lo4, 4);
 465                 memcpy(buf+4, &hi4, 4);
 466         }
 467         else {
 468                 memcpy(buf, &hi4, 4);
 469                 memcpy(buf+4, &lo4, 4);
 470         }
 471         return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 472 #endif
 473 }
 474
 475 static PyObject *
 476 r_object(RFILE *p)
 477 {
 478         /* NULL is a valid return value, it does not necessarily means that
 479            an exception is set. */
 480         PyObject *v, *v2, *v3;
 481         long i, n;
 482         int type = r_byte(p);
 483
 484         switch (type) {
 485
 486         case EOF:
 487                 PyErr_SetString(PyExc_EOFError,
 488                                 "EOF read where object expected");
 489                 return NULL;
 490
 491         case TYPE_NULL:
 492                 return NULL;
 493
 494         case TYPE_NONE:
 495                 Py_INCREF(Py_None);
 496                 return Py_None;
 497
 498         case TYPE_STOPITER:
 499                 Py_INCREF(PyExc_StopIteration);
 500                 return PyExc_StopIteration;
 501
 502         case TYPE_ELLIPSIS:
 503                 Py_INCREF(Py_Ellipsis);
 504                 return Py_Ellipsis;
 505
 506         case TYPE_FALSE:
 507                 Py_INCREF(Py_False);
 508                 return Py_False;
 509
 510         case TYPE_TRUE:
 511                 Py_INCREF(Py_True);
 512                 return Py_True;
 513
 514         case TYPE_INT:
 515                 return PyInt_FromLong(r_long(p));
 516
 517         case TYPE_INT64:
 518                 return r_long64(p);
 519
 520         case TYPE_LONG:
 521                 {
 522                         int size;
 523                         PyLongObject *ob;
 524                         n = r_long(p);
 525                         size = n<0 ? -n : n;
 526                         ob = _PyLong_New(size);
 527                         if (ob == NULL)
 528                                 return NULL;
 529                         ob->ob_size = n;
 530                         for (i = 0; i < size; i++) {
 531                                 int digit = r_short(p);
 532                                 if (digit < 0) {
 533                                         Py_DECREF(ob);
 534                                         PyErr_SetString(PyExc_ValueError,
 535                                                         "bad marshal data");
 536                                         return NULL;
 537                                 }
 538                                 ob->ob_digit[i] = digit;
 539                         }
 540                         return (PyObject *)ob;
 541                 }
 542
 543         case TYPE_FLOAT:
 544                 {
 545                         char buf[256];
 546                         double dx;
 547                         n = r_byte(p);
 548                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 549                                 PyErr_SetString(PyExc_EOFError,
 550                                         "EOF read where object expected");
 551                                 return NULL;
 552                         }
 553                         buf[n] = '\0';
 554                         PyFPE_START_PROTECT("atof", return 0)
 555                         dx = PyOS_ascii_atof(buf);
 556                         PyFPE_END_PROTECT(dx)
 557                         return PyFloat_FromDouble(dx);
 558                 }
 559
 560         case TYPE_BINARY_FLOAT:
 561                 {
 562                         unsigned char buf[8];
 563                         double x;
 564                         if (r_string((char*)buf, 8, p) != 8) {
 565                                 PyErr_SetString(PyExc_EOFError,
 566                                         "EOF read where object expected");
 567                                 return NULL;
 568                         }
 569                         x = _PyFloat_Unpack8(buf, 1);
 570                         if (x == -1.0 && PyErr_Occurred()) {
 571                                 return NULL;
 572                         }
 573                         return PyFloat_FromDouble(x);
 574                 }
 575
 576 #ifndef WITHOUT_COMPLEX
 577         case TYPE_COMPLEX:
 578                 {
 579                         char buf[256];
 580                         Py_complex c;
 581                         n = r_byte(p);
 582                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 583                                 PyErr_SetString(PyExc_EOFError,
 584                                         "EOF read where object expected");
 585                                 return NULL;
 586                         }
 587                         buf[n] = '\0';
 588                         PyFPE_START_PROTECT("atof", return 0)
 589                         c.real = PyOS_ascii_atof(buf);
 590                         PyFPE_END_PROTECT(c)
 591                         n = r_byte(p);
 592                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 593                                 PyErr_SetString(PyExc_EOFError,
 594                                         "EOF read where object expected");
 595                                 return NULL;
 596                         }
 597                         buf[n] = '\0';
 598                         PyFPE_START_PROTECT("atof", return 0)
 599                         c.imag = PyOS_ascii_atof(buf);
 600                         PyFPE_END_PROTECT(c)
 601                         return PyComplex_FromCComplex(c);
 602                 }
 603
 604         case TYPE_BINARY_COMPLEX:
 605                 {
 606                         unsigned char buf[8];
 607                         Py_complex c;
 608                         if (r_string((char*)buf, 8, p) != 8) {
 609                                 PyErr_SetString(PyExc_EOFError,
 610                                         "EOF read where object expected");
 611                                 return NULL;
 612                         }
 613                         c.real = _PyFloat_Unpack8(buf, 1);
 614                         if (c.real == -1.0 && PyErr_Occurred()) {
 615                                 return NULL;
 616                         }
 617                         if (r_string((char*)buf, 8, p) != 8) {
 618                                 PyErr_SetString(PyExc_EOFError,
 619                                         "EOF read where object expected");
 620                                 return NULL;
 621                         }
 622                         c.imag = _PyFloat_Unpack8(buf, 1);
 623                         if (c.imag == -1.0 && PyErr_Occurred()) {
 624                                 return NULL;
 625                         }
 626                         return PyComplex_FromCComplex(c);
 627                 }
 628 #endif
 629
 630         case TYPE_INTERNED:
 631         case TYPE_STRING:
 632                 n = r_long(p);
 633                 if (n < 0) {
 634                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 635                         return NULL;
 636                 }
 637                 v = PyString_FromStringAndSize((char *)NULL, n);
 638                 if (v == NULL)
 639                         return v;
 640                 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
 641                         Py_DECREF(v);
 642                         PyErr_SetString(PyExc_EOFError,
 643                                         "EOF read where object expected");
 644                         return NULL;
 645                 }
 646                 if (type == TYPE_INTERNED) {
 647                         PyString_InternInPlace(&v);
 648                         PyList_Append(p->strings, v);
 649                 }
 650                 return v;
 651
 652         case TYPE_STRINGREF:
 653                 n = r_long(p);
 654                 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
 655                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 656                         return NULL;
 657                 }
 658                 v = PyList_GET_ITEM(p->strings, n);
 659                 Py_INCREF(v);
 660                 return v;
 661
 662 #ifdef Py_USING_UNICODE
 663         case TYPE_UNICODE:
 664             {
 665                 char *buffer;
 666
 667                 n = r_long(p);
 668                 if (n < 0) {
 669                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 670                         return NULL;
 671                 }
 672                 buffer = PyMem_NEW(char, n);
 673                 if (buffer == NULL)
 674                         return PyErr_NoMemory();
 675                 if (r_string(buffer, (int)n, p) != n) {
 676                         PyMem_DEL(buffer);
 677                         PyErr_SetString(PyExc_EOFError,
 678                                 "EOF read where object expected");
 679                         return NULL;
 680                 }
 681                 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
 682                 PyMem_DEL(buffer);
 683                 return v;
 684             }
 685 #endif
 686
 687         case TYPE_TUPLE:
 688                 n = r_long(p);
 689                 if (n < 0) {
 690                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 691                         return NULL;
 692                 }
 693                 v = PyTuple_New((int)n);
 694                 if (v == NULL)
 695                         return v;
 696                 for (i = 0; i < n; i++) {
 697                         v2 = r_object(p);
 698                         if ( v2 == NULL ) {
 699                                 if (!PyErr_Occurred())
 700                                         PyErr_SetString(PyExc_TypeError,
 701                                                 "NULL object in marshal data");
 702                                 Py_DECREF(v);
 703                                 v = NULL;
 704                                 break;
 705                         }
 706                         PyTuple_SET_ITEM(v, (int)i, v2);
 707                 }
 708                 return v;
 709
 710         case TYPE_LIST:
 711                 n = r_long(p);
 712                 if (n < 0) {
 713                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 714                         return NULL;
 715                 }
 716                 v = PyList_New((int)n);
 717                 if (v == NULL)
 718                         return v;
 719                 for (i = 0; i < n; i++) {
 720                         v2 = r_object(p);
 721                         if ( v2 == NULL ) {
 722                                 if (!PyErr_Occurred())
 723                                         PyErr_SetString(PyExc_TypeError,
 724                                                 "NULL object in marshal data");
 725                                 Py_DECREF(v);
 726                                 v = NULL;
 727                                 break;
 728                         }
 729                         PyList_SetItem(v, (int)i, v2);
 730                 }
 731                 return v;
 732
 733         case TYPE_DICT:
 734                 v = PyDict_New();
 735                 if (v == NULL)
 736                         return NULL;
 737                 for (;;) {
 738                         PyObject *key, *val;
 739                         key = r_object(p);
 740                         if (key == NULL)
 741                                 break;
 742                         val = r_object(p);
 743                         if (val != NULL)
 744                                 PyDict_SetItem(v, key, val);
 745                         Py_DECREF(key);
 746                         Py_XDECREF(val);
 747                 }
 748                 if (PyErr_Occurred()) {
 749                         Py_DECREF(v);
 750                         v = NULL;
 751                 }
 752                 return v;
 753
 754         case TYPE_SET:
 755         case TYPE_FROZENSET:
 756                 n = r_long(p);
 757                 if (n < 0) {
 758                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 759                         return NULL;
 760                 }
 761                 v = PyTuple_New((int)n);
 762                 if (v == NULL)
 763                         return v;
 764                 for (i = 0; i < n; i++) {
 765                         v2 = r_object(p);
 766                         if ( v2 == NULL ) {
 767                                 if (!PyErr_Occurred())
 768                                         PyErr_SetString(PyExc_TypeError,
 769                                                 "NULL object in marshal data");
 770                                 Py_DECREF(v);
 771                                 v = NULL;
 772                                 break;
 773                         }
 774                         PyTuple_SET_ITEM(v, (int)i, v2);
 775                 }
 776                 if (v == NULL)
 777                         return v;
 778                 if (type == TYPE_SET)
 779                         v3 = PySet_New(v);
 780                 else
 781                         v3 = PyFrozenSet_New(v);
 782                 Py_DECREF(v);
 783                 return v3;
 784
 785         case TYPE_CODE:
 786                 if (PyEval_GetRestricted()) {
 787                         PyErr_SetString(PyExc_RuntimeError,
 788                                 "cannot unmarshal code objects in "
 789                                 "restricted execution mode");
 790                         return NULL;
 791                 }
 792                 else {
 793                         int argcount;
 794                         int nlocals;
 795                         int stacksize;
 796                         int flags;
 797                         PyObject *code = NULL;
 798                         PyObject *consts = NULL;
 799                         PyObject *names = NULL;
 800                         PyObject *varnames = NULL;
 801                         PyObject *freevars = NULL;
 802                         PyObject *cellvars = NULL;
 803                         PyObject *filename = NULL;
 804                         PyObject *name = NULL;
 805                         int firstlineno;
 806                         PyObject *lnotab = NULL;
 807
 808                         v = NULL;
 809
 810                         argcount = r_long(p);
 811                         nlocals = r_long(p);
 812                         stacksize = r_long(p);
 813                         flags = r_long(p);
 814                         code = r_object(p);
 815                         if (code == NULL)
 816                                 goto code_error;
 817                         consts = r_object(p);
 818                         if (consts == NULL)
 819                                 goto code_error;
 820                         names = r_object(p);
 821                         if (names == NULL)
 822                                 goto code_error;
 823                         varnames = r_object(p);
 824                         if (varnames == NULL)
 825                                 goto code_error;
 826                         freevars = r_object(p);
 827                         if (freevars == NULL)
 828                                 goto code_error;
 829                         cellvars = r_object(p);
 830                         if (cellvars == NULL)
 831                                 goto code_error;
 832                         filename = r_object(p);
 833                         if (filename == NULL)
 834                                 goto code_error;
 835                         name = r_object(p);
 836                         if (name == NULL)
 837                                 goto code_error;
 838                         firstlineno = r_long(p);
 839                         lnotab = r_object(p);
 840                         if (lnotab == NULL)
 841                                 goto code_error;
 842
 843                         v = (PyObject *) PyCode_New(
 844                                         argcount, nlocals, stacksize, flags,
 845                                         code, consts, names, varnames,
 846                                         freevars, cellvars, filename, name,
 847                                         firstlineno, lnotab);
 848
 849                   code_error:
 850                         Py_XDECREF(code);
 851                         Py_XDECREF(consts);
 852                         Py_XDECREF(names);
 853                         Py_XDECREF(varnames);
 854                         Py_XDECREF(freevars);
 855                         Py_XDECREF(cellvars);
 856                         Py_XDECREF(filename);
 857                         Py_XDECREF(name);
 858                         Py_XDECREF(lnotab);
 859
 860                 }
 861                 return v;
 862
 863         default:
 864                 /* Bogus data got written, which isn't ideal.
 865                    This will let you keep working and recover. */
 866                 PyErr_SetString(PyExc_ValueError, "bad marshal data");
 867                 return NULL;
 868
 869         }
 870 }
 871
 872 static PyObject *
 873 read_object(RFILE *p)
 874 {
 875         PyObject *v;
 876         if (PyErr_Occurred()) {
 877                 fprintf(stderr, "XXX readobject called with exception set\n");
 878                 return NULL;
 879         }
 880         v = r_object(p);
 881         if (v == NULL && !PyErr_Occurred())
 882                 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
 883         return v;
 884 }
 885
 886 int
 887 PyMarshal_ReadShortFromFile(FILE *fp)
 888 {
 889         RFILE rf;
 890         assert(fp);
 891         rf.fp = fp;
 892         rf.strings = NULL;
 893         rf.end = rf.ptr = NULL;
 894         return r_short(&rf);
 895 }
 896
 897 long
 898 PyMarshal_ReadLongFromFile(FILE *fp)
 899 {
 900         RFILE rf;
 901         rf.fp = fp;
 902         rf.strings = NULL;
 903         return r_long(&rf);
 904 }
 905
 906 #ifdef HAVE_FSTAT
 907 /* Return size of file in bytes; < 0 if unknown. */
 908 static off_t
 909 getfilesize(FILE *fp)
 910 {
 911         struct stat st;
 912         if (fstat(fileno(fp), &st) != 0)
 913                 return -1;
 914         else
 915                 return st.st_size;
 916 }
 917 #endif
 918
 919 /* If we can get the size of the file up-front, and it's reasonably small,
 920  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
 921  * than reading a byte at a time from file; speeds .pyc imports.
 922  * CAUTION:  since this may read the entire remainder of the file, don't
 923  * call it unless you know you're done with the file.
 924  */
 925 PyObject *
 926 PyMarshal_ReadLastObjectFromFile(FILE *fp)
 927 {
 928 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
 929  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
 930  */
 931 #define SMALL_FILE_LIMIT (1L << 14)
 932 #define REASONABLE_FILE_LIMIT (1L << 18)
 933 #ifdef HAVE_FSTAT
 934         off_t filesize;
 935 #endif
 936 #ifdef HAVE_FSTAT
 937         filesize = getfilesize(fp);
 938         if (filesize > 0) {
 939                 char buf[SMALL_FILE_LIMIT];
 940                 char* pBuf = NULL;
 941                 if (filesize <= SMALL_FILE_LIMIT)
 942                         pBuf = buf;
 943                 else if (filesize <= REASONABLE_FILE_LIMIT)
 944                         pBuf = (char *)PyMem_MALLOC(filesize);
 945                 if (pBuf != NULL) {
 946                         PyObject* v;
 947                         size_t n;
 948                         /* filesize must fit into an int, because it
 949                            is smaller than REASONABLE_FILE_LIMIT */
 950                         n = fread(pBuf, 1, (int)filesize, fp);
 951                         v = PyMarshal_ReadObjectFromString(pBuf, n);
 952                         if (pBuf != buf)
 953                                 PyMem_FREE(pBuf);
 954                         return v;
 955                 }
 956
 957         }
 958 #endif
 959         /* We don't have fstat, or we do but the file is larger than
 960          * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
 961          */
 962         return PyMarshal_ReadObjectFromFile(fp);
 963
 964 #undef SMALL_FILE_LIMIT
 965 #undef REASONABLE_FILE_LIMIT
 966 }
 967
 968 PyObject *
 969 PyMarshal_ReadObjectFromFile(FILE *fp)
 970 {
 971         RFILE rf;
 972         PyObject *result;
 973         rf.fp = fp;
 974         rf.strings = PyList_New(0);
 975         result = r_object(&rf);
 976         Py_DECREF(rf.strings);
 977         return result;
 978 }
 979
 980 PyObject *
 981 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
 982 {
 983         RFILE rf;
 984         PyObject *result;
 985         rf.fp = NULL;
 986         rf.ptr = str;
 987         rf.end = str + len;
 988         rf.strings = PyList_New(0);
 989         result = r_object(&rf);
 990         Py_DECREF(rf.strings);
 991         return result;
 992 }
 993
 994 PyObject *
 995 PyMarshal_WriteObjectToString(PyObject *x, int version)
 996 {
 997         WFILE wf;
 998         wf.fp = NULL;
 999         wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1000         if (wf.str == NULL)
1001                 return NULL;
1002         wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1003         wf.end = wf.ptr + PyString_Size(wf.str);
1004         wf.error = 0;
1005         wf.depth = 0;
1006         wf.version = version;
1007         wf.strings = (version > 0) ? PyDict_New() : NULL;
1008         w_object(x, &wf);
1009         Py_XDECREF(wf.strings);
1010         if (wf.str != NULL)
1011                 _PyString_Resize(&wf.str,
1012                     (int) (wf.ptr -
1013                            PyString_AS_STRING((PyStringObject *)wf.str)));
1014         if (wf.error) {
1015                 Py_XDECREF(wf.str);
1016                 PyErr_SetString(PyExc_ValueError,
1017                                 (wf.error==1)?"unmarshallable object"
1018                                 :"object too deeply nested to marshal");
1019                 return NULL;
1020         }
1021         return wf.str;
1022 }
1023
1024 /* And an interface for Python programs... */
1025
1026 static PyObject *
1027 marshal_dump(PyObject *self, PyObject *args)
1028 {
1029         WFILE wf;
1030         PyObject *x;
1031         PyObject *f;
1032         int version = Py_MARSHAL_VERSION;
1033         if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1034                 return NULL;
1035         if (!PyFile_Check(f)) {
1036                 PyErr_SetString(PyExc_TypeError,
1037                                 "marshal.dump() 2nd arg must be file");
1038                 return NULL;
1039         }
1040         wf.fp = PyFile_AsFile(f);
1041         wf.str = NULL;
1042         wf.ptr = wf.end = NULL;
1043         wf.error = 0;
1044         wf.depth = 0;
1045         wf.strings = (version > 0) ? PyDict_New() : 0;
1046         wf.version = version;
1047         w_object(x, &wf);
1048         Py_XDECREF(wf.strings);
1049         if (wf.error) {
1050                 PyErr_SetString(PyExc_ValueError,
1051                                 (wf.error==1)?"unmarshallable object"
1052                                 :"object too deeply nested to marshal");
1053                 return NULL;
1054         }
1055         Py_INCREF(Py_None);
1056         return Py_None;
1057 }
1058
1059 static PyObject *
1060 marshal_load(PyObject *self, PyObject *args)
1061 {
1062         RFILE rf;
1063         PyObject *f, *result;
1064         if (!PyArg_ParseTuple(args, "O:load", &f))
1065                 return NULL;
1066         if (!PyFile_Check(f)) {
1067                 PyErr_SetString(PyExc_TypeError,
1068                                 "marshal.load() arg must be file");
1069                 return NULL;
1070         }
1071         rf.fp = PyFile_AsFile(f);
1072         rf.strings = PyList_New(0);
1073         result = read_object(&rf);
1074         Py_DECREF(rf.strings);
1075         return result;
1076 }
1077
1078 static PyObject *
1079 marshal_dumps(PyObject *self, PyObject *args)
1080 {
1081         PyObject *x;
1082         int version = Py_MARSHAL_VERSION;
1083         if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1084                 return NULL;
1085         return PyMarshal_WriteObjectToString(x, version);
1086 }
1087
1088 static PyObject *
1089 marshal_loads(PyObject *self, PyObject *args)
1090 {
1091         RFILE rf;
1092         char *s;
1093         Py_ssize_t n;
1094         PyObject* result;
1095         if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1096                 return NULL;
1097         rf.fp = NULL;
1098         rf.ptr = s;
1099         rf.end = s + n;
1100         rf.strings = PyList_New(0);
1101         result = read_object(&rf);
1102         Py_DECREF(rf.strings);
1103         return result;
1104 }
1105
1106 static PyMethodDef marshal_methods[] = {
1107         {"dump",        marshal_dump,   METH_VARARGS},
1108         {"load",        marshal_load,   METH_VARARGS},
1109         {"dumps",       marshal_dumps,  METH_VARARGS},
1110         {"loads",       marshal_loads,  METH_VARARGS},
1111         {NULL,          NULL}           /* sentinel */
1112 };
1113
1114 PyMODINIT_FUNC
1115 PyMarshal_Init(void)
1116 {
1117         PyObject *mod = Py_InitModule("marshal", marshal_methods);
1118         if (mod == NULL)
1119                 return;
1120         PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1121 }