Python/marshal.c

   1
   2 /* Write Python objects to files and read them back.
   3    This is intended for writing and reading compiled Python code only;
   4    a true persistent storage facility would be much harder, since
   5    it would have to take circular links and sharing into account. */
   6
   7 #define PY_SSIZE_T_CLEAN
   8
   9 #include "Python.h"
  10 #include "longintrepr.h"
  11 #include "code.h"
  12 #include "marshal.h"
  13
  14 /* High water mark to determine when the marshalled object is dangerously deep
  15  * and risks coring the interpreter.  When the object stack gets this deep,
  16  * raise an exception instead of continuing.
  17  */
  18 #define MAX_MARSHAL_STACK_DEPTH 2000
  19
  20 #define TYPE_NULL               '0'
  21 #define TYPE_NONE               'N'
  22 #define TYPE_FALSE              'F'
  23 #define TYPE_TRUE               'T'
  24 #define TYPE_STOPITER           'S'
  25 #define TYPE_ELLIPSIS           '.'
  26 #define TYPE_INT                'i'
  27 #define TYPE_INT64              'I'
  28 #define TYPE_FLOAT              'f'
  29 #define TYPE_BINARY_FLOAT       'g'
  30 #define TYPE_COMPLEX            'x'
  31 #define TYPE_BINARY_COMPLEX     'y'
  32 #define TYPE_LONG               'l'
  33 #define TYPE_STRING             's'
  34 #define TYPE_INTERNED           't'
  35 #define TYPE_STRINGREF          'R'
  36 #define TYPE_TUPLE              '('
  37 #define TYPE_LIST               '['
  38 #define TYPE_DICT               '{'
  39 #define TYPE_CODE               'c'
  40 #define TYPE_UNICODE            'u'
  41 #define TYPE_UNKNOWN            '?'
  42 #define TYPE_SET                '<'
  43 #define TYPE_FROZENSET          '>'
  44
  45 typedef struct {
  46         FILE *fp;
  47         int error;
  48         int depth;
  49         /* If fp == NULL, the following are valid: */
  50         PyObject *str;
  51         char *ptr;
  52         char *end;
  53         PyObject *strings; /* dict on marshal, list on unmarshal */
  54         int version;
  55 } WFILE;
  56
  57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  58                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  59                            else w_more(c, p)
  60
  61 static void
  62 w_more(int c, WFILE *p)
  63 {
  64         Py_ssize_t size, newsize;
  65         if (p->str == NULL)
  66                 return; /* An error already occurred */
  67         size = PyString_Size(p->str);
  68         newsize = size + size + 1024;
  69         if (newsize > 32*1024*1024) {
  70                 newsize = size + (size >> 3);   /* 12.5% overallocation */
  71         }
  72         if (_PyString_Resize(&p->str, newsize) != 0) {
  73                 p->ptr = p->end = NULL;
  74         }
  75         else {
  76                 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
  77                 p->end =
  78                         PyString_AS_STRING((PyStringObject *)p->str) + newsize;
  79                 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
  80         }
  81 }
  82
  83 static void
  84 w_string(char *s, int n, WFILE *p)
  85 {
  86         if (p->fp != NULL) {
  87                 fwrite(s, 1, n, p->fp);
  88         }
  89         else {
  90                 while (--n >= 0) {
  91                         w_byte(*s, p);
  92                         s++;
  93                 }
  94         }
  95 }
  96
  97 static void
  98 w_short(int x, WFILE *p)
  99 {
 100         w_byte((char)( x      & 0xff), p);
 101         w_byte((char)((x>> 8) & 0xff), p);
 102 }
 103
 104 static void
 105 w_long(long x, WFILE *p)
 106 {
 107         w_byte((char)( x      & 0xff), p);
 108         w_byte((char)((x>> 8) & 0xff), p);
 109         w_byte((char)((x>>16) & 0xff), p);
 110         w_byte((char)((x>>24) & 0xff), p);
 111 }
 112
 113 #if SIZEOF_LONG > 4
 114 static void
 115 w_long64(long x, WFILE *p)
 116 {
 117         w_long(x, p);
 118         w_long(x>>32, p);
 119 }
 120 #endif
 121
 122 static void
 123 w_object(PyObject *v, WFILE *p)
 124 {
 125         Py_ssize_t i, n;
 126
 127         p->depth++;
 128
 129         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 130                 p->error = 2;
 131         }
 132         else if (v == NULL) {
 133                 w_byte(TYPE_NULL, p);
 134         }
 135         else if (v == Py_None) {
 136                 w_byte(TYPE_NONE, p);
 137         }
 138         else if (v == PyExc_StopIteration) {
 139                 w_byte(TYPE_STOPITER, p);
 140         }
 141         else if (v == Py_Ellipsis) {
 142                 w_byte(TYPE_ELLIPSIS, p);
 143         }
 144         else if (v == Py_False) {
 145                 w_byte(TYPE_FALSE, p);
 146         }
 147         else if (v == Py_True) {
 148                 w_byte(TYPE_TRUE, p);
 149         }
 150         else if (PyInt_CheckExact(v)) {
 151                 long x = PyInt_AS_LONG((PyIntObject *)v);
 152 #if SIZEOF_LONG > 4
 153                 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
 154                 if (y && y != -1) {
 155                         w_byte(TYPE_INT64, p);
 156                         w_long64(x, p);
 157                 }
 158                 else
 159 #endif
 160                         {
 161                         w_byte(TYPE_INT, p);
 162                         w_long(x, p);
 163                 }
 164         }
 165         else if (PyLong_CheckExact(v)) {
 166                 PyLongObject *ob = (PyLongObject *)v;
 167                 w_byte(TYPE_LONG, p);
 168                 n = ob->ob_size;
 169                 w_long((long)n, p);
 170                 if (n < 0)
 171                         n = -n;
 172                 for (i = 0; i < n; i++)
 173                         w_short(ob->ob_digit[i], p);
 174         }
 175         else if (PyFloat_CheckExact(v)) {
 176                 if (p->version > 1) {
 177                         unsigned char buf[8];
 178                         if (_PyFloat_Pack8(PyFloat_AsDouble(v),
 179                                            buf, 1) < 0) {
 180                                 p->error = 1;
 181                                 return;
 182                         }
 183                         w_byte(TYPE_BINARY_FLOAT, p);
 184                         w_string((char*)buf, 8, p);
 185                 }
 186                 else {
 187                         char buf[256]; /* Plenty to format any double */
 188                         PyFloat_AsReprString(buf, (PyFloatObject *)v);
 189                         n = strlen(buf);
 190                         w_byte(TYPE_FLOAT, p);
 191                         w_byte((int)n, p);
 192                         w_string(buf, (int)n, p);
 193                 }
 194         }
 195 #ifndef WITHOUT_COMPLEX
 196         else if (PyComplex_CheckExact(v)) {
 197                 if (p->version > 1) {
 198                         unsigned char buf[8];
 199                         if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
 200                                            buf, 1) < 0) {
 201                                 p->error = 1;
 202                                 return;
 203                         }
 204                         w_byte(TYPE_BINARY_COMPLEX, p);
 205                         w_string((char*)buf, 8, p);
 206                         if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
 207                                            buf, 1) < 0) {
 208                                 p->error = 1;
 209                                 return;
 210                         }
 211                         w_string((char*)buf, 8, p);
 212                 }
 213                 else {
 214                         char buf[256]; /* Plenty to format any double */
 215                         PyFloatObject *temp;
 216                         w_byte(TYPE_COMPLEX, p);
 217                         temp = (PyFloatObject*)PyFloat_FromDouble(
 218                                 PyComplex_RealAsDouble(v));
 219                         if (!temp) {
 220                                 p->error = 1;
 221                                 return;
 222                         }
 223                         PyFloat_AsReprString(buf, temp);
 224                         Py_DECREF(temp);
 225                         n = strlen(buf);
 226                         w_byte((int)n, p);
 227                         w_string(buf, (int)n, p);
 228                         temp = (PyFloatObject*)PyFloat_FromDouble(
 229                                 PyComplex_ImagAsDouble(v));
 230                         if (!temp) {
 231                                 p->error = 1;
 232                                 return;
 233                         }
 234                         PyFloat_AsReprString(buf, temp);
 235                         Py_DECREF(temp);
 236                         n = strlen(buf);
 237                         w_byte((int)n, p);
 238                         w_string(buf, (int)n, p);
 239                 }
 240         }
 241 #endif
 242         else if (PyString_CheckExact(v)) {
 243                 if (p->strings && PyString_CHECK_INTERNED(v)) {
 244                         PyObject *o = PyDict_GetItem(p->strings, v);
 245                         if (o) {
 246                                 long w = PyInt_AsLong(o);
 247                                 w_byte(TYPE_STRINGREF, p);
 248                                 w_long(w, p);
 249                                 goto exit;
 250                         }
 251                         else {
 252                                 int ok;
 253                                 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
 254                                 ok = o &&
 255                                      PyDict_SetItem(p->strings, v, o) >= 0;
 256                                 Py_XDECREF(o);
 257                                 if (!ok) {
 258                                         p->depth--;
 259                                         p->error = 1;
 260                                         return;
 261                                 }
 262                                 w_byte(TYPE_INTERNED, p);
 263                         }
 264                 }
 265                 else {
 266                         w_byte(TYPE_STRING, p);
 267                 }
 268                 n = PyString_GET_SIZE(v);
 269                 if (n > INT_MAX) {
 270                         /* huge strings are not supported */
 271                         p->depth--;
 272                         p->error = 1;
 273                         return;
 274                 }
 275                 w_long((long)n, p);
 276                 w_string(PyString_AS_STRING(v), (int)n, p);
 277         }
 278 #ifdef Py_USING_UNICODE
 279         else if (PyUnicode_CheckExact(v)) {
 280                 PyObject *utf8;
 281                 utf8 = PyUnicode_AsUTF8String(v);
 282                 if (utf8 == NULL) {
 283                         p->depth--;
 284                         p->error = 1;
 285                         return;
 286                 }
 287                 w_byte(TYPE_UNICODE, p);
 288                 n = PyString_GET_SIZE(utf8);
 289                 if (n > INT_MAX) {
 290                         p->depth--;
 291                         p->error = 1;
 292                         return;
 293                 }
 294                 w_long((long)n, p);
 295                 w_string(PyString_AS_STRING(utf8), (int)n, p);
 296                 Py_DECREF(utf8);
 297         }
 298 #endif
 299         else if (PyTuple_CheckExact(v)) {
 300                 w_byte(TYPE_TUPLE, p);
 301                 n = PyTuple_Size(v);
 302                 w_long((long)n, p);
 303                 for (i = 0; i < n; i++) {
 304                         w_object(PyTuple_GET_ITEM(v, i), p);
 305                 }
 306         }
 307         else if (PyList_CheckExact(v)) {
 308                 w_byte(TYPE_LIST, p);
 309                 n = PyList_GET_SIZE(v);
 310                 w_long((long)n, p);
 311                 for (i = 0; i < n; i++) {
 312                         w_object(PyList_GET_ITEM(v, i), p);
 313                 }
 314         }
 315         else if (PyDict_CheckExact(v)) {
 316                 Py_ssize_t pos;
 317                 PyObject *key, *value;
 318                 w_byte(TYPE_DICT, p);
 319                 /* This one is NULL object terminated! */
 320                 pos = 0;
 321                 while (PyDict_Next(v, &pos, &key, &value)) {
 322                         w_object(key, p);
 323                         w_object(value, p);
 324                 }
 325                 w_object((PyObject *)NULL, p);
 326         }
 327         else if (PyAnySet_CheckExact(v)) {
 328                 PyObject *value, *it;
 329
 330                 if (PyObject_TypeCheck(v, &PySet_Type))
 331                         w_byte(TYPE_SET, p);
 332                 else
 333                         w_byte(TYPE_FROZENSET, p);
 334                 n = PyObject_Size(v);
 335                 if (n == -1) {
 336                         p->depth--;
 337                         p->error = 1;
 338                         return;
 339                 }
 340                 w_long((long)n, p);
 341                 it = PyObject_GetIter(v);
 342                 if (it == NULL) {
 343                         p->depth--;
 344                         p->error = 1;
 345                         return;
 346                 }
 347                 while ((value = PyIter_Next(it)) != NULL) {
 348                         w_object(value, p);
 349                         Py_DECREF(value);
 350                 }
 351                 Py_DECREF(it);
 352                 if (PyErr_Occurred()) {
 353                         p->depth--;
 354                         p->error = 1;
 355                         return;
 356                 }
 357         }
 358         else if (PyCode_Check(v)) {
 359                 PyCodeObject *co = (PyCodeObject *)v;
 360                 w_byte(TYPE_CODE, p);
 361                 w_long(co->co_argcount, p);
 362                 w_long(co->co_nlocals, p);
 363                 w_long(co->co_stacksize, p);
 364                 w_long(co->co_flags, p);
 365                 w_object(co->co_code, p);
 366                 w_object(co->co_consts, p);
 367                 w_object(co->co_names, p);
 368                 w_object(co->co_varnames, p);
 369                 w_object(co->co_freevars, p);
 370                 w_object(co->co_cellvars, p);
 371                 w_object(co->co_filename, p);
 372                 w_object(co->co_name, p);
 373                 w_long(co->co_firstlineno, p);
 374                 w_object(co->co_lnotab, p);
 375         }
 376         else if (PyObject_CheckReadBuffer(v)) {
 377                 /* Write unknown buffer-style objects as a string */
 378                 char *s;
 379                 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
 380                 w_byte(TYPE_STRING, p);
 381                 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
 382                 if (n > INT_MAX) {
 383                         p->depth--;
 384                         p->error = 1;
 385                         return;
 386                 }
 387                 w_long((long)n, p);
 388                 w_string(s, (int)n, p);
 389         }
 390         else {
 391                 w_byte(TYPE_UNKNOWN, p);
 392                 p->error = 1;
 393         }
 394    exit:
 395         p->depth--;
 396 }
 397
 398 /* version currently has no effect for writing longs. */
 399 void
 400 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 401 {
 402         WFILE wf;
 403         wf.fp = fp;
 404         wf.error = 0;
 405         wf.depth = 0;
 406         wf.strings = NULL;
 407         wf.version = version;
 408         w_long(x, &wf);
 409 }
 410
 411 void
 412 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 413 {
 414         WFILE wf;
 415         wf.fp = fp;
 416         wf.error = 0;
 417         wf.depth = 0;
 418         wf.strings = (version > 0) ? PyDict_New() : NULL;
 419         wf.version = version;
 420         w_object(x, &wf);
 421         Py_XDECREF(wf.strings);
 422 }
 423
 424 typedef WFILE RFILE; /* Same struct with different invariants */
 425
 426 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 427
 428 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 429
 430 static int
 431 r_string(char *s, int n, RFILE *p)
 432 {
 433         if (p->fp != NULL)
 434                 /* The result fits into int because it must be <=n. */
 435                 return (int)fread(s, 1, n, p->fp);
 436         if (p->end - p->ptr < n)
 437                 n = (int)(p->end - p->ptr);
 438         memcpy(s, p->ptr, n);
 439         p->ptr += n;
 440         return n;
 441 }
 442
 443 static int
 444 r_short(RFILE *p)
 445 {
 446         register short x;
 447         x = r_byte(p);
 448         x |= r_byte(p) << 8;
 449         /* Sign-extension, in case short greater than 16 bits */
 450         x |= -(x & 0x8000);
 451         return x;
 452 }
 453
 454 static long
 455 r_long(RFILE *p)
 456 {
 457         register long x;
 458         register FILE *fp = p->fp;
 459         if (fp) {
 460                 x = getc(fp);
 461                 x |= (long)getc(fp) << 8;
 462                 x |= (long)getc(fp) << 16;
 463                 x |= (long)getc(fp) << 24;
 464         }
 465         else {
 466                 x = rs_byte(p);
 467                 x |= (long)rs_byte(p) << 8;
 468                 x |= (long)rs_byte(p) << 16;
 469                 x |= (long)rs_byte(p) << 24;
 470         }
 471 #if SIZEOF_LONG > 4
 472         /* Sign extension for 64-bit machines */
 473         x |= -(x & 0x80000000L);
 474 #endif
 475         return x;
 476 }
 477
 478 /* r_long64 deals with the TYPE_INT64 code.  On a machine with
 479    sizeof(long) > 4, it returns a Python int object, else a Python long
 480    object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
 481    so there's no inefficiency here in returning a PyLong on 32-bit boxes
 482    for everything written via TYPE_INT64 (i.e., if an int is written via
 483    TYPE_INT64, it *needs* more than 32 bits).
 484 */
 485 static PyObject *
 486 r_long64(RFILE *p)
 487 {
 488         long lo4 = r_long(p);
 489         long hi4 = r_long(p);
 490 #if SIZEOF_LONG > 4
 491         long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
 492         return PyInt_FromLong(x);
 493 #else
 494         unsigned char buf[8];
 495         int one = 1;
 496         int is_little_endian = (int)*(char*)&one;
 497         if (is_little_endian) {
 498                 memcpy(buf, &lo4, 4);
 499                 memcpy(buf+4, &hi4, 4);
 500         }
 501         else {
 502                 memcpy(buf, &hi4, 4);
 503                 memcpy(buf+4, &lo4, 4);
 504         }
 505         return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 506 #endif
 507 }
 508
 509 static PyObject *
 510 r_object(RFILE *p)
 511 {
 512         /* NULL is a valid return value, it does not necessarily means that
 513            an exception is set. */
 514         PyObject *v, *v2;
 515         long i, n;
 516         int type = r_byte(p);
 517         PyObject *retval;
 518
 519         p->depth++;
 520
 521         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 522                 p->depth--;
 523                 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
 524                 return NULL;
 525         }
 526
 527         switch (type) {
 528
 529         case EOF:
 530                 PyErr_SetString(PyExc_EOFError,
 531                                 "EOF read where object expected");
 532                 retval = NULL;
 533                 break;
 534
 535         case TYPE_NULL:
 536                 retval = NULL;
 537                 break;
 538
 539         case TYPE_NONE:
 540                 Py_INCREF(Py_None);
 541                 retval = Py_None;
 542                 break;
 543
 544         case TYPE_STOPITER:
 545                 Py_INCREF(PyExc_StopIteration);
 546                 retval = PyExc_StopIteration;
 547                 break;
 548
 549         case TYPE_ELLIPSIS:
 550                 Py_INCREF(Py_Ellipsis);
 551                 retval = Py_Ellipsis;
 552                 break;
 553
 554         case TYPE_FALSE:
 555                 Py_INCREF(Py_False);
 556                 retval = Py_False;
 557                 break;
 558
 559         case TYPE_TRUE:
 560                 Py_INCREF(Py_True);
 561                 retval = Py_True;
 562                 break;
 563
 564         case TYPE_INT:
 565                 retval = PyInt_FromLong(r_long(p));
 566                 break;
 567
 568         case TYPE_INT64:
 569                 retval = r_long64(p);
 570                 break;
 571
 572         case TYPE_LONG:
 573                 {
 574                         int size;
 575                         PyLongObject *ob;
 576                         n = r_long(p);
 577                         if (n < -INT_MAX || n > INT_MAX) {
 578                                 PyErr_SetString(PyExc_ValueError,
 579                                                 "bad marshal data");
 580                                 retval = NULL;
 581                                 break;
 582                         }
 583                         size = n<0 ? -n : n;
 584                         ob = _PyLong_New(size);
 585                         if (ob == NULL) {
 586                                 retval = NULL;
 587                                 break;
 588                         }
 589                         ob->ob_size = n;
 590                         for (i = 0; i < size; i++) {
 591                                 int digit = r_short(p);
 592                                 if (digit < 0) {
 593                                         Py_DECREF(ob);
 594                                         PyErr_SetString(PyExc_ValueError,
 595                                                         "bad marshal data");
 596                                         ob = NULL;
 597                                         break;
 598                                 }
 599                                 if (ob != NULL)
 600                                         ob->ob_digit[i] = digit;
 601                         }
 602                         retval = (PyObject *)ob;
 603                         break;
 604                 }
 605
 606         case TYPE_FLOAT:
 607                 {
 608                         char buf[256];
 609                         double dx;
 610                         n = r_byte(p);
 611                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 612                                 PyErr_SetString(PyExc_EOFError,
 613                                         "EOF read where object expected");
 614                                 retval = NULL;
 615                                 break;
 616                         }
 617                         buf[n] = '\0';
 618                         retval = NULL;
 619                         PyFPE_START_PROTECT("atof", break)
 620                         dx = PyOS_ascii_atof(buf);
 621                         PyFPE_END_PROTECT(dx)
 622                         retval = PyFloat_FromDouble(dx);
 623                         break;
 624                 }
 625
 626         case TYPE_BINARY_FLOAT:
 627                 {
 628                         unsigned char buf[8];
 629                         double x;
 630                         if (r_string((char*)buf, 8, p) != 8) {
 631                                 PyErr_SetString(PyExc_EOFError,
 632                                         "EOF read where object expected");
 633                                 retval = NULL;
 634                                 break;
 635                         }
 636                         x = _PyFloat_Unpack8(buf, 1);
 637                         if (x == -1.0 && PyErr_Occurred()) {
 638                                 retval = NULL;
 639                                 break;
 640                         }
 641                         retval = PyFloat_FromDouble(x);
 642                         break;
 643                 }
 644
 645 #ifndef WITHOUT_COMPLEX
 646         case TYPE_COMPLEX:
 647                 {
 648                         char buf[256];
 649                         Py_complex c;
 650                         n = r_byte(p);
 651                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 652                                 PyErr_SetString(PyExc_EOFError,
 653                                         "EOF read where object expected");
 654                                 retval = NULL;
 655                                 break;
 656                         }
 657                         buf[n] = '\0';
 658                         retval = NULL;
 659                         PyFPE_START_PROTECT("atof", break;)
 660                         c.real = PyOS_ascii_atof(buf);
 661                         PyFPE_END_PROTECT(c)
 662                         n = r_byte(p);
 663                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 664                                 PyErr_SetString(PyExc_EOFError,
 665                                         "EOF read where object expected");
 666                                 retval = NULL;
 667                                 break;
 668                         }
 669                         buf[n] = '\0';
 670                         PyFPE_START_PROTECT("atof", break)
 671                         c.imag = PyOS_ascii_atof(buf);
 672                         PyFPE_END_PROTECT(c)
 673                         retval = PyComplex_FromCComplex(c);
 674                         break;
 675                 }
 676
 677         case TYPE_BINARY_COMPLEX:
 678                 {
 679                         unsigned char buf[8];
 680                         Py_complex c;
 681                         if (r_string((char*)buf, 8, p) != 8) {
 682                                 PyErr_SetString(PyExc_EOFError,
 683                                         "EOF read where object expected");
 684                                 retval = NULL;
 685                                 break;
 686                         }
 687                         c.real = _PyFloat_Unpack8(buf, 1);
 688                         if (c.real == -1.0 && PyErr_Occurred()) {
 689                                 retval = NULL;
 690                                 break;
 691                         }
 692                         if (r_string((char*)buf, 8, p) != 8) {
 693                                 PyErr_SetString(PyExc_EOFError,
 694                                         "EOF read where object expected");
 695                                 retval = NULL;
 696                                 break;
 697                         }
 698                         c.imag = _PyFloat_Unpack8(buf, 1);
 699                         if (c.imag == -1.0 && PyErr_Occurred()) {
 700                                 retval = NULL;
 701                                 break;
 702                         }
 703                         retval = PyComplex_FromCComplex(c);
 704                         break;
 705                 }
 706 #endif
 707
 708         case TYPE_INTERNED:
 709         case TYPE_STRING:
 710                 n = r_long(p);
 711                 if (n < 0 || n > INT_MAX) {
 712                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 713                         retval = NULL;
 714                         break;
 715                 }
 716                 v = PyString_FromStringAndSize((char *)NULL, n);
 717                 if (v == NULL) {
 718                         retval = NULL;
 719                         break;
 720                 }
 721                 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
 722                         Py_DECREF(v);
 723                         PyErr_SetString(PyExc_EOFError,
 724                                         "EOF read where object expected");
 725                         retval = NULL;
 726                         break;
 727                 }
 728                 if (type == TYPE_INTERNED) {
 729                         PyString_InternInPlace(&v);
 730                         if (PyList_Append(p->strings, v) < 0) {
 731                                 retval = NULL;
 732                                 break;
 733                         }
 734                 }
 735                 retval = v;
 736                 break;
 737
 738         case TYPE_STRINGREF:
 739                 n = r_long(p);
 740                 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
 741                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 742                         retval = NULL;
 743                         break;
 744                 }
 745                 v = PyList_GET_ITEM(p->strings, n);
 746                 Py_INCREF(v);
 747                 retval = v;
 748                 break;
 749
 750 #ifdef Py_USING_UNICODE
 751         case TYPE_UNICODE:
 752             {
 753                 char *buffer;
 754
 755                 n = r_long(p);
 756                 if (n < 0 || n > INT_MAX) {
 757                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 758                         retval = NULL;
 759                         break;
 760                 }
 761                 buffer = PyMem_NEW(char, n);
 762                 if (buffer == NULL) {
 763                         retval = PyErr_NoMemory();
 764                         break;
 765                 }
 766                 if (r_string(buffer, (int)n, p) != n) {
 767                         PyMem_DEL(buffer);
 768                         PyErr_SetString(PyExc_EOFError,
 769                                 "EOF read where object expected");
 770                         retval = NULL;
 771                         break;
 772                 }
 773                 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
 774                 PyMem_DEL(buffer);
 775                 retval = v;
 776                 break;
 777             }
 778 #endif
 779
 780         case TYPE_TUPLE:
 781                 n = r_long(p);
 782                 if (n < 0 || n > INT_MAX) {
 783                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 784                         retval = NULL;
 785                         break;
 786                 }
 787                 v = PyTuple_New((int)n);
 788                 if (v == NULL) {
 789                         retval = NULL;
 790                         break;
 791                 }
 792                 for (i = 0; i < n; i++) {
 793                         v2 = r_object(p);
 794                         if ( v2 == NULL ) {
 795                                 if (!PyErr_Occurred())
 796                                         PyErr_SetString(PyExc_TypeError,
 797                                                 "NULL object in marshal data");
 798                                 Py_DECREF(v);
 799                                 v = NULL;
 800                                 break;
 801                         }
 802                         PyTuple_SET_ITEM(v, (int)i, v2);
 803                 }
 804                 retval = v;
 805                 break;
 806
 807         case TYPE_LIST:
 808                 n = r_long(p);
 809                 if (n < 0 || n > INT_MAX) {
 810                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 811                         retval = NULL;
 812                         break;
 813                 }
 814                 v = PyList_New((int)n);
 815                 if (v == NULL) {
 816                         retval = NULL;
 817                         break;
 818                 }
 819                 for (i = 0; i < n; i++) {
 820                         v2 = r_object(p);
 821                         if ( v2 == NULL ) {
 822                                 if (!PyErr_Occurred())
 823                                         PyErr_SetString(PyExc_TypeError,
 824                                                 "NULL object in marshal data");
 825                                 Py_DECREF(v);
 826                                 v = NULL;
 827                                 break;
 828                         }
 829                         PyList_SET_ITEM(v, (int)i, v2);
 830                 }
 831                 retval = v;
 832                 break;
 833
 834         case TYPE_DICT:
 835                 v = PyDict_New();
 836                 if (v == NULL) {
 837                         retval = NULL;
 838                         break;
 839                 }
 840                 for (;;) {
 841                         PyObject *key, *val;
 842                         key = r_object(p);
 843                         if (key == NULL)
 844                                 break;
 845                         val = r_object(p);
 846                         if (val != NULL)
 847                                 PyDict_SetItem(v, key, val);
 848                         Py_DECREF(key);
 849                         Py_XDECREF(val);
 850                 }
 851                 if (PyErr_Occurred()) {
 852                         Py_DECREF(v);
 853                         v = NULL;
 854                 }
 855                 retval = v;
 856                 break;
 857
 858         case TYPE_SET:
 859         case TYPE_FROZENSET:
 860                 n = r_long(p);
 861                 if (n < 0 || n > INT_MAX) {
 862                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 863                         retval = NULL;
 864                         break;
 865                 }
 866                 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
 867                 if (v == NULL) {
 868                         retval = NULL;
 869                         break;
 870                 }
 871                 for (i = 0; i < n; i++) {
 872                         v2 = r_object(p);
 873                         if ( v2 == NULL ) {
 874                                 if (!PyErr_Occurred())
 875                                         PyErr_SetString(PyExc_TypeError,
 876                                                 "NULL object in marshal data");
 877                                 Py_DECREF(v);
 878                                 v = NULL;
 879                                 break;
 880                         }
 881                         if (PySet_Add(v, v2) == -1) {
 882                                 Py_DECREF(v);
 883                                 Py_DECREF(v2);
 884                                 v = NULL;
 885                                 break;
 886                         }
 887                         Py_DECREF(v2);
 888                 }
 889                 retval = v;
 890                 break;
 891
 892         case TYPE_CODE:
 893                 if (PyEval_GetRestricted()) {
 894                         PyErr_SetString(PyExc_RuntimeError,
 895                                 "cannot unmarshal code objects in "
 896                                 "restricted execution mode");
 897                         retval = NULL;
 898                         break;
 899                 }
 900                 else {
 901                         int argcount;
 902                         int nlocals;
 903                         int stacksize;
 904                         int flags;
 905                         PyObject *code = NULL;
 906                         PyObject *consts = NULL;
 907                         PyObject *names = NULL;
 908                         PyObject *varnames = NULL;
 909                         PyObject *freevars = NULL;
 910                         PyObject *cellvars = NULL;
 911                         PyObject *filename = NULL;
 912                         PyObject *name = NULL;
 913                         int firstlineno;
 914                         PyObject *lnotab = NULL;
 915
 916                         v = NULL;
 917
 918                         /* XXX ignore long->int overflows for now */
 919                         argcount = (int)r_long(p);
 920                         nlocals = (int)r_long(p);
 921                         stacksize = (int)r_long(p);
 922                         flags = (int)r_long(p);
 923                         code = r_object(p);
 924                         if (code == NULL)
 925                                 goto code_error;
 926                         consts = r_object(p);
 927                         if (consts == NULL)
 928                                 goto code_error;
 929                         names = r_object(p);
 930                         if (names == NULL)
 931                                 goto code_error;
 932                         varnames = r_object(p);
 933                         if (varnames == NULL)
 934                                 goto code_error;
 935                         freevars = r_object(p);
 936                         if (freevars == NULL)
 937                                 goto code_error;
 938                         cellvars = r_object(p);
 939                         if (cellvars == NULL)
 940                                 goto code_error;
 941                         filename = r_object(p);
 942                         if (filename == NULL)
 943                                 goto code_error;
 944                         name = r_object(p);
 945                         if (name == NULL)
 946                                 goto code_error;
 947                         firstlineno = (int)r_long(p);
 948                         lnotab = r_object(p);
 949                         if (lnotab == NULL)
 950                                 goto code_error;
 951
 952                         v = (PyObject *) PyCode_New(
 953                                         argcount, nlocals, stacksize, flags,
 954                                         code, consts, names, varnames,
 955                                         freevars, cellvars, filename, name,
 956                                         firstlineno, lnotab);
 957
 958                   code_error:
 959                         Py_XDECREF(code);
 960                         Py_XDECREF(consts);
 961                         Py_XDECREF(names);
 962                         Py_XDECREF(varnames);
 963                         Py_XDECREF(freevars);
 964                         Py_XDECREF(cellvars);
 965                         Py_XDECREF(filename);
 966                         Py_XDECREF(name);
 967                         Py_XDECREF(lnotab);
 968
 969                 }
 970                 retval = v;
 971                 break;
 972
 973         default:
 974                 /* Bogus data got written, which isn't ideal.
 975                    This will let you keep working and recover. */
 976                 PyErr_SetString(PyExc_ValueError, "bad marshal data");
 977                 retval = NULL;
 978                 break;
 979
 980         }
 981         p->depth--;
 982         return retval;
 983 }
 984
 985 static PyObject *
 986 read_object(RFILE *p)
 987 {
 988         PyObject *v;
 989         if (PyErr_Occurred()) {
 990                 fprintf(stderr, "XXX readobject called with exception set\n");
 991                 return NULL;
 992         }
 993         v = r_object(p);
 994         if (v == NULL && !PyErr_Occurred())
 995                 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
 996         return v;
 997 }
 998
 999 int
1000 PyMarshal_ReadShortFromFile(FILE *fp)
1001 {
1002         RFILE rf;
1003         assert(fp);
1004         rf.fp = fp;
1005         rf.strings = NULL;
1006         rf.end = rf.ptr = NULL;
1007         return r_short(&rf);
1008 }
1009
1010 long
1011 PyMarshal_ReadLongFromFile(FILE *fp)
1012 {
1013         RFILE rf;
1014         rf.fp = fp;
1015         rf.strings = NULL;
1016         rf.ptr = rf.end = NULL;
1017         return r_long(&rf);
1018 }
1019
1020 #ifdef HAVE_FSTAT
1021 /* Return size of file in bytes; < 0 if unknown. */
1022 static off_t
1023 getfilesize(FILE *fp)
1024 {
1025         struct stat st;
1026         if (fstat(fileno(fp), &st) != 0)
1027                 return -1;
1028         else
1029                 return st.st_size;
1030 }
1031 #endif
1032
1033 /* If we can get the size of the file up-front, and it's reasonably small,
1034  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
1035  * than reading a byte at a time from file; speeds .pyc imports.
1036  * CAUTION:  since this may read the entire remainder of the file, don't
1037  * call it unless you know you're done with the file.
1038  */
1039 PyObject *
1040 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1041 {
1042 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1043  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1044  */
1045 #define SMALL_FILE_LIMIT (1L << 14)
1046 #define REASONABLE_FILE_LIMIT (1L << 18)
1047 #ifdef HAVE_FSTAT
1048         off_t filesize;
1049 #endif
1050 #ifdef HAVE_FSTAT
1051         filesize = getfilesize(fp);
1052         if (filesize > 0) {
1053                 char buf[SMALL_FILE_LIMIT];
1054                 char* pBuf = NULL;
1055                 if (filesize <= SMALL_FILE_LIMIT)
1056                         pBuf = buf;
1057                 else if (filesize <= REASONABLE_FILE_LIMIT)
1058                         pBuf = (char *)PyMem_MALLOC(filesize);
1059                 if (pBuf != NULL) {
1060                         PyObject* v;
1061                         size_t n;
1062                         /* filesize must fit into an int, because it
1063                            is smaller than REASONABLE_FILE_LIMIT */
1064                         n = fread(pBuf, 1, (int)filesize, fp);
1065                         v = PyMarshal_ReadObjectFromString(pBuf, n);
1066                         if (pBuf != buf)
1067                                 PyMem_FREE(pBuf);
1068                         return v;
1069                 }
1070
1071         }
1072 #endif
1073         /* We don't have fstat, or we do but the file is larger than
1074          * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1075          */
1076         return PyMarshal_ReadObjectFromFile(fp);
1077
1078 #undef SMALL_FILE_LIMIT
1079 #undef REASONABLE_FILE_LIMIT
1080 }
1081
1082 PyObject *
1083 PyMarshal_ReadObjectFromFile(FILE *fp)
1084 {
1085         RFILE rf;
1086         PyObject *result;
1087         rf.fp = fp;
1088         rf.strings = PyList_New(0);
1089         rf.depth = 0;
1090         rf.ptr = rf.end = NULL;
1091         result = r_object(&rf);
1092         Py_DECREF(rf.strings);
1093         return result;
1094 }
1095
1096 PyObject *
1097 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1098 {
1099         RFILE rf;
1100         PyObject *result;
1101         rf.fp = NULL;
1102         rf.ptr = str;
1103         rf.end = str + len;
1104         rf.strings = PyList_New(0);
1105         rf.depth = 0;
1106         result = r_object(&rf);
1107         Py_DECREF(rf.strings);
1108         return result;
1109 }
1110
1111 PyObject *
1112 PyMarshal_WriteObjectToString(PyObject *x, int version)
1113 {
1114         WFILE wf;
1115         wf.fp = NULL;
1116         wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1117         if (wf.str == NULL)
1118                 return NULL;
1119         wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1120         wf.end = wf.ptr + PyString_Size(wf.str);
1121         wf.error = 0;
1122         wf.depth = 0;
1123         wf.version = version;
1124         wf.strings = (version > 0) ? PyDict_New() : NULL;
1125         w_object(x, &wf);
1126         Py_XDECREF(wf.strings);
1127         if (wf.str != NULL) {
1128                 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1129                 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1130                         Py_DECREF(wf.str);
1131                         PyErr_SetString(PyExc_OverflowError,
1132                                         "too much marshall data for a string");
1133                         return NULL;
1134                 }
1135                 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1136         }
1137         if (wf.error) {
1138                 Py_XDECREF(wf.str);
1139                 PyErr_SetString(PyExc_ValueError,
1140                                 (wf.error==1)?"unmarshallable object"
1141                                 :"object too deeply nested to marshal");
1142                 return NULL;
1143         }
1144         return wf.str;
1145 }
1146
1147 /* And an interface for Python programs... */
1148
1149 static PyObject *
1150 marshal_dump(PyObject *self, PyObject *args)
1151 {
1152         WFILE wf;
1153         PyObject *x;
1154         PyObject *f;
1155         int version = Py_MARSHAL_VERSION;
1156         if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1157                 return NULL;
1158         if (!PyFile_Check(f)) {
1159                 PyErr_SetString(PyExc_TypeError,
1160                                 "marshal.dump() 2nd arg must be file");
1161                 return NULL;
1162         }
1163         wf.fp = PyFile_AsFile(f);
1164         wf.str = NULL;
1165         wf.ptr = wf.end = NULL;
1166         wf.error = 0;
1167         wf.depth = 0;
1168         wf.strings = (version > 0) ? PyDict_New() : 0;
1169         wf.version = version;
1170         w_object(x, &wf);
1171         Py_XDECREF(wf.strings);
1172         if (wf.error) {
1173                 PyErr_SetString(PyExc_ValueError,
1174                                 (wf.error==1)?"unmarshallable object"
1175                                 :"object too deeply nested to marshal");
1176                 return NULL;
1177         }
1178         Py_INCREF(Py_None);
1179         return Py_None;
1180 }
1181
1182 static PyObject *
1183 marshal_load(PyObject *self, PyObject *f)
1184 {
1185         RFILE rf;
1186         PyObject *result;
1187         if (!PyFile_Check(f)) {
1188                 PyErr_SetString(PyExc_TypeError,
1189                                 "marshal.load() arg must be file");
1190                 return NULL;
1191         }
1192         rf.fp = PyFile_AsFile(f);
1193         rf.strings = PyList_New(0);
1194         rf.depth = 0;
1195         result = read_object(&rf);
1196         Py_DECREF(rf.strings);
1197         return result;
1198 }
1199
1200 static PyObject *
1201 marshal_dumps(PyObject *self, PyObject *args)
1202 {
1203         PyObject *x;
1204         int version = Py_MARSHAL_VERSION;
1205         if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1206                 return NULL;
1207         return PyMarshal_WriteObjectToString(x, version);
1208 }
1209
1210 static PyObject *
1211 marshal_loads(PyObject *self, PyObject *args)
1212 {
1213         RFILE rf;
1214         char *s;
1215         Py_ssize_t n;
1216         PyObject* result;
1217         if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1218                 return NULL;
1219         rf.fp = NULL;
1220         rf.ptr = s;
1221         rf.end = s + n;
1222         rf.strings = PyList_New(0);
1223         rf.depth = 0;
1224         result = read_object(&rf);
1225         Py_DECREF(rf.strings);
1226         return result;
1227 }
1228
1229 static PyMethodDef marshal_methods[] = {
1230         {"dump",        marshal_dump,   METH_VARARGS},
1231         {"load",        marshal_load,   METH_O},
1232         {"dumps",       marshal_dumps,  METH_VARARGS},
1233         {"loads",       marshal_loads,  METH_VARARGS},
1234         {NULL,          NULL}           /* sentinel */
1235 };
1236
1237 PyMODINIT_FUNC
1238 PyMarshal_Init(void)
1239 {
1240         PyObject *mod = Py_InitModule("marshal", marshal_methods);
1241         if (mod == NULL)
1242                 return;
1243         PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1244 }