Python/marshal.c

   1
   2 /* Write Python objects to files and read them back.
   3    This is intended for writing and reading compiled Python code only;
   4    a true persistent storage facility would be much harder, since
   5    it would have to take circular links and sharing into account. */
   6
   7 #define PY_SSIZE_T_CLEAN
   8
   9 #include "Python.h"
  10 #include "longintrepr.h"
  11 #include "code.h"
  12 #include "marshal.h"
  13
  14 /* High water mark to determine when the marshalled object is dangerously deep
  15  * and risks coring the interpreter.  When the object stack gets this deep,
  16  * raise an exception instead of continuing.
  17  */
  18 #define MAX_MARSHAL_STACK_DEPTH 2000
  19
  20 #define TYPE_NULL               '0'
  21 #define TYPE_NONE               'N'
  22 #define TYPE_FALSE              'F'
  23 #define TYPE_TRUE               'T'
  24 #define TYPE_STOPITER           'S'
  25 #define TYPE_ELLIPSIS           '.'
  26 #define TYPE_INT                'i'
  27 #define TYPE_INT64              'I'
  28 #define TYPE_FLOAT              'f'
  29 #define TYPE_BINARY_FLOAT       'g'
  30 #define TYPE_COMPLEX            'x'
  31 #define TYPE_BINARY_COMPLEX     'y'
  32 #define TYPE_LONG               'l'
  33 #define TYPE_STRING             's'
  34 #define TYPE_INTERNED           't'
  35 #define TYPE_STRINGREF          'R'
  36 #define TYPE_TUPLE              '('
  37 #define TYPE_LIST               '['
  38 #define TYPE_DICT               '{'
  39 #define TYPE_CODE               'c'
  40 #define TYPE_UNICODE            'u'
  41 #define TYPE_UNKNOWN            '?'
  42 #define TYPE_SET                '<'
  43 #define TYPE_FROZENSET          '>'
  44
  45 typedef struct {
  46         FILE *fp;
  47         int error;
  48         int depth;
  49         /* If fp == NULL, the following are valid: */
  50         PyObject *str;
  51         char *ptr;
  52         char *end;
  53         PyObject *strings; /* dict on marshal, list on unmarshal */
  54         int version;
  55 } WFILE;
  56
  57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  58                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  59                            else w_more(c, p)
  60
  61 static void
  62 w_more(int c, WFILE *p)
  63 {
  64         Py_ssize_t size, newsize;
  65         if (p->str == NULL)
  66                 return; /* An error already occurred */
  67         size = PyString_Size(p->str);
  68         newsize = size + 1024;
  69         if (_PyString_Resize(&p->str, newsize) != 0) {
  70                 p->ptr = p->end = NULL;
  71         }
  72         else {
  73                 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
  74                 p->end =
  75                         PyString_AS_STRING((PyStringObject *)p->str) + newsize;
  76                 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
  77         }
  78 }
  79
  80 static void
  81 w_string(char *s, int n, WFILE *p)
  82 {
  83         if (p->fp != NULL) {
  84                 fwrite(s, 1, n, p->fp);
  85         }
  86         else {
  87                 while (--n >= 0) {
  88                         w_byte(*s, p);
  89                         s++;
  90                 }
  91         }
  92 }
  93
  94 static void
  95 w_short(int x, WFILE *p)
  96 {
  97         w_byte((char)( x      & 0xff), p);
  98         w_byte((char)((x>> 8) & 0xff), p);
  99 }
 100
 101 static void
 102 w_long(long x, WFILE *p)
 103 {
 104         w_byte((char)( x      & 0xff), p);
 105         w_byte((char)((x>> 8) & 0xff), p);
 106         w_byte((char)((x>>16) & 0xff), p);
 107         w_byte((char)((x>>24) & 0xff), p);
 108 }
 109
 110 #if SIZEOF_LONG > 4
 111 static void
 112 w_long64(long x, WFILE *p)
 113 {
 114         w_long(x, p);
 115         w_long(x>>32, p);
 116 }
 117 #endif
 118
 119 static void
 120 w_object(PyObject *v, WFILE *p)
 121 {
 122         Py_ssize_t i, n;
 123
 124         p->depth++;
 125
 126         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 127                 p->error = 2;
 128         }
 129         else if (v == NULL) {
 130                 w_byte(TYPE_NULL, p);
 131         }
 132         else if (v == Py_None) {
 133                 w_byte(TYPE_NONE, p);
 134         }
 135         else if (v == PyExc_StopIteration) {
 136                 w_byte(TYPE_STOPITER, p);
 137         }
 138         else if (v == Py_Ellipsis) {
 139                 w_byte(TYPE_ELLIPSIS, p);
 140         }
 141         else if (v == Py_False) {
 142                 w_byte(TYPE_FALSE, p);
 143         }
 144         else if (v == Py_True) {
 145                 w_byte(TYPE_TRUE, p);
 146         }
 147         else if (PyInt_CheckExact(v)) {
 148                 long x = PyInt_AS_LONG((PyIntObject *)v);
 149 #if SIZEOF_LONG > 4
 150                 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
 151                 if (y && y != -1) {
 152                         w_byte(TYPE_INT64, p);
 153                         w_long64(x, p);
 154                 }
 155                 else
 156 #endif
 157                         {
 158                         w_byte(TYPE_INT, p);
 159                         w_long(x, p);
 160                 }
 161         }
 162         else if (PyLong_CheckExact(v)) {
 163                 PyLongObject *ob = (PyLongObject *)v;
 164                 w_byte(TYPE_LONG, p);
 165                 n = ob->ob_size;
 166                 w_long((long)n, p);
 167                 if (n < 0)
 168                         n = -n;
 169                 for (i = 0; i < n; i++)
 170                         w_short(ob->ob_digit[i], p);
 171         }
 172         else if (PyFloat_CheckExact(v)) {
 173                 if (p->version > 1) {
 174                         unsigned char buf[8];
 175                         if (_PyFloat_Pack8(PyFloat_AsDouble(v),
 176                                            buf, 1) < 0) {
 177                                 p->error = 1;
 178                                 return;
 179                         }
 180                         w_byte(TYPE_BINARY_FLOAT, p);
 181                         w_string((char*)buf, 8, p);
 182                 }
 183                 else {
 184                         char buf[256]; /* Plenty to format any double */
 185                         PyFloat_AsReprString(buf, (PyFloatObject *)v);
 186                         n = strlen(buf);
 187                         w_byte(TYPE_FLOAT, p);
 188                         w_byte((int)n, p);
 189                         w_string(buf, (int)n, p);
 190                 }
 191         }
 192 #ifndef WITHOUT_COMPLEX
 193         else if (PyComplex_CheckExact(v)) {
 194                 if (p->version > 1) {
 195                         unsigned char buf[8];
 196                         if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
 197                                            buf, 1) < 0) {
 198                                 p->error = 1;
 199                                 return;
 200                         }
 201                         w_byte(TYPE_BINARY_COMPLEX, p);
 202                         w_string((char*)buf, 8, p);
 203                         if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
 204                                            buf, 1) < 0) {
 205                                 p->error = 1;
 206                                 return;
 207                         }
 208                         w_string((char*)buf, 8, p);
 209                 }
 210                 else {
 211                         char buf[256]; /* Plenty to format any double */
 212                         PyFloatObject *temp;
 213                         w_byte(TYPE_COMPLEX, p);
 214                         temp = (PyFloatObject*)PyFloat_FromDouble(
 215                                 PyComplex_RealAsDouble(v));
 216                         if (!temp) {
 217                                 p->error = 1;
 218                                 return;
 219                         }
 220                         PyFloat_AsReprString(buf, temp);
 221                         Py_DECREF(temp);
 222                         n = strlen(buf);
 223                         w_byte((int)n, p);
 224                         w_string(buf, (int)n, p);
 225                         temp = (PyFloatObject*)PyFloat_FromDouble(
 226                                 PyComplex_ImagAsDouble(v));
 227                         if (!temp) {
 228                                 p->error = 1;
 229                                 return;
 230                         }
 231                         PyFloat_AsReprString(buf, temp);
 232                         Py_DECREF(temp);
 233                         n = strlen(buf);
 234                         w_byte((int)n, p);
 235                         w_string(buf, (int)n, p);
 236                 }
 237         }
 238 #endif
 239         else if (PyString_CheckExact(v)) {
 240                 if (p->strings && PyString_CHECK_INTERNED(v)) {
 241                         PyObject *o = PyDict_GetItem(p->strings, v);
 242                         if (o) {
 243                                 long w = PyInt_AsLong(o);
 244                                 w_byte(TYPE_STRINGREF, p);
 245                                 w_long(w, p);
 246                                 goto exit;
 247                         }
 248                         else {
 249                                 int ok;
 250                                 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
 251                                 ok = o &&
 252                                      PyDict_SetItem(p->strings, v, o) >= 0;
 253                                 Py_XDECREF(o);
 254                                 if (!ok) {
 255                                         p->depth--;
 256                                         p->error = 1;
 257                                         return;
 258                                 }
 259                                 w_byte(TYPE_INTERNED, p);
 260                         }
 261                 }
 262                 else {
 263                         w_byte(TYPE_STRING, p);
 264                 }
 265                 n = PyString_GET_SIZE(v);
 266                 if (n > INT_MAX) {
 267                         /* huge strings are not supported */
 268                         p->depth--;
 269                         p->error = 1;
 270                         return;
 271                 }
 272                 w_long((long)n, p);
 273                 w_string(PyString_AS_STRING(v), (int)n, p);
 274         }
 275 #ifdef Py_USING_UNICODE
 276         else if (PyUnicode_CheckExact(v)) {
 277                 PyObject *utf8;
 278                 utf8 = PyUnicode_AsUTF8String(v);
 279                 if (utf8 == NULL) {
 280                         p->depth--;
 281                         p->error = 1;
 282                         return;
 283                 }
 284                 w_byte(TYPE_UNICODE, p);
 285                 n = PyString_GET_SIZE(utf8);
 286                 if (n > INT_MAX) {
 287                         p->depth--;
 288                         p->error = 1;
 289                         return;
 290                 }
 291                 w_long((long)n, p);
 292                 w_string(PyString_AS_STRING(utf8), (int)n, p);
 293                 Py_DECREF(utf8);
 294         }
 295 #endif
 296         else if (PyTuple_CheckExact(v)) {
 297                 w_byte(TYPE_TUPLE, p);
 298                 n = PyTuple_Size(v);
 299                 w_long((long)n, p);
 300                 for (i = 0; i < n; i++) {
 301                         w_object(PyTuple_GET_ITEM(v, i), p);
 302                 }
 303         }
 304         else if (PyList_CheckExact(v)) {
 305                 w_byte(TYPE_LIST, p);
 306                 n = PyList_GET_SIZE(v);
 307                 w_long((long)n, p);
 308                 for (i = 0; i < n; i++) {
 309                         w_object(PyList_GET_ITEM(v, i), p);
 310                 }
 311         }
 312         else if (PyDict_CheckExact(v)) {
 313                 Py_ssize_t pos;
 314                 PyObject *key, *value;
 315                 w_byte(TYPE_DICT, p);
 316                 /* This one is NULL object terminated! */
 317                 pos = 0;
 318                 while (PyDict_Next(v, &pos, &key, &value)) {
 319                         w_object(key, p);
 320                         w_object(value, p);
 321                 }
 322                 w_object((PyObject *)NULL, p);
 323         }
 324         else if (PyAnySet_CheckExact(v)) {
 325                 PyObject *value, *it;
 326
 327                 if (PyObject_TypeCheck(v, &PySet_Type))
 328                         w_byte(TYPE_SET, p);
 329                 else
 330                         w_byte(TYPE_FROZENSET, p);
 331                 n = PyObject_Size(v);
 332                 if (n == -1) {
 333                         p->depth--;
 334                         p->error = 1;
 335                         return;
 336                 }
 337                 w_long((long)n, p);
 338                 it = PyObject_GetIter(v);
 339                 if (it == NULL) {
 340                         p->depth--;
 341                         p->error = 1;
 342                         return;
 343                 }
 344                 while ((value = PyIter_Next(it)) != NULL) {
 345                         w_object(value, p);
 346                         Py_DECREF(value);
 347                 }
 348                 Py_DECREF(it);
 349                 if (PyErr_Occurred()) {
 350                         p->depth--;
 351                         p->error = 1;
 352                         return;
 353                 }
 354         }
 355         else if (PyCode_Check(v)) {
 356                 PyCodeObject *co = (PyCodeObject *)v;
 357                 w_byte(TYPE_CODE, p);
 358                 w_long(co->co_argcount, p);
 359                 w_long(co->co_nlocals, p);
 360                 w_long(co->co_stacksize, p);
 361                 w_long(co->co_flags, p);
 362                 w_object(co->co_code, p);
 363                 w_object(co->co_consts, p);
 364                 w_object(co->co_names, p);
 365                 w_object(co->co_varnames, p);
 366                 w_object(co->co_freevars, p);
 367                 w_object(co->co_cellvars, p);
 368                 w_object(co->co_filename, p);
 369                 w_object(co->co_name, p);
 370                 w_long(co->co_firstlineno, p);
 371                 w_object(co->co_lnotab, p);
 372         }
 373         else if (PyObject_CheckReadBuffer(v)) {
 374                 /* Write unknown buffer-style objects as a string */
 375                 char *s;
 376                 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
 377                 w_byte(TYPE_STRING, p);
 378                 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
 379                 if (n > INT_MAX) {
 380                         p->depth--;
 381                         p->error = 1;
 382                         return;
 383                 }
 384                 w_long((long)n, p);
 385                 w_string(s, (int)n, p);
 386         }
 387         else {
 388                 w_byte(TYPE_UNKNOWN, p);
 389                 p->error = 1;
 390         }
 391    exit:
 392         p->depth--;
 393 }
 394
 395 /* version currently has no effect for writing longs. */
 396 void
 397 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 398 {
 399         WFILE wf;
 400         wf.fp = fp;
 401         wf.error = 0;
 402         wf.depth = 0;
 403         wf.strings = NULL;
 404         wf.version = version;
 405         w_long(x, &wf);
 406 }
 407
 408 void
 409 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 410 {
 411         WFILE wf;
 412         wf.fp = fp;
 413         wf.error = 0;
 414         wf.depth = 0;
 415         wf.strings = (version > 0) ? PyDict_New() : NULL;
 416         wf.version = version;
 417         w_object(x, &wf);
 418         Py_XDECREF(wf.strings);
 419 }
 420
 421 typedef WFILE RFILE; /* Same struct with different invariants */
 422
 423 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 424
 425 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 426
 427 static int
 428 r_string(char *s, int n, RFILE *p)
 429 {
 430         if (p->fp != NULL)
 431                 /* The result fits into int because it must be <=n. */
 432                 return (int)fread(s, 1, n, p->fp);
 433         if (p->end - p->ptr < n)
 434                 n = (int)(p->end - p->ptr);
 435         memcpy(s, p->ptr, n);
 436         p->ptr += n;
 437         return n;
 438 }
 439
 440 static int
 441 r_short(RFILE *p)
 442 {
 443         register short x;
 444         x = r_byte(p);
 445         x |= r_byte(p) << 8;
 446         /* Sign-extension, in case short greater than 16 bits */
 447         x |= -(x & 0x8000);
 448         return x;
 449 }
 450
 451 static long
 452 r_long(RFILE *p)
 453 {
 454         register long x;
 455         register FILE *fp = p->fp;
 456         if (fp) {
 457                 x = getc(fp);
 458                 x |= (long)getc(fp) << 8;
 459                 x |= (long)getc(fp) << 16;
 460                 x |= (long)getc(fp) << 24;
 461         }
 462         else {
 463                 x = rs_byte(p);
 464                 x |= (long)rs_byte(p) << 8;
 465                 x |= (long)rs_byte(p) << 16;
 466                 x |= (long)rs_byte(p) << 24;
 467         }
 468 #if SIZEOF_LONG > 4
 469         /* Sign extension for 64-bit machines */
 470         x |= -(x & 0x80000000L);
 471 #endif
 472         return x;
 473 }
 474
 475 /* r_long64 deals with the TYPE_INT64 code.  On a machine with
 476    sizeof(long) > 4, it returns a Python int object, else a Python long
 477    object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
 478    so there's no inefficiency here in returning a PyLong on 32-bit boxes
 479    for everything written via TYPE_INT64 (i.e., if an int is written via
 480    TYPE_INT64, it *needs* more than 32 bits).
 481 */
 482 static PyObject *
 483 r_long64(RFILE *p)
 484 {
 485         long lo4 = r_long(p);
 486         long hi4 = r_long(p);
 487 #if SIZEOF_LONG > 4
 488         long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
 489         return PyInt_FromLong(x);
 490 #else
 491         unsigned char buf[8];
 492         int one = 1;
 493         int is_little_endian = (int)*(char*)&one;
 494         if (is_little_endian) {
 495                 memcpy(buf, &lo4, 4);
 496                 memcpy(buf+4, &hi4, 4);
 497         }
 498         else {
 499                 memcpy(buf, &hi4, 4);
 500                 memcpy(buf+4, &lo4, 4);
 501         }
 502         return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 503 #endif
 504 }
 505
 506 static PyObject *
 507 r_object(RFILE *p)
 508 {
 509         /* NULL is a valid return value, it does not necessarily means that
 510            an exception is set. */
 511         PyObject *v, *v2;
 512         long i, n;
 513         int type = r_byte(p);
 514         PyObject *retval;
 515
 516         p->depth++;
 517
 518         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 519                 p->depth--;
 520                 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
 521                 return NULL;
 522         }
 523
 524         switch (type) {
 525
 526         case EOF:
 527                 PyErr_SetString(PyExc_EOFError,
 528                                 "EOF read where object expected");
 529                 retval = NULL;
 530                 break;
 531
 532         case TYPE_NULL:
 533                 retval = NULL;
 534                 break;
 535
 536         case TYPE_NONE:
 537                 Py_INCREF(Py_None);
 538                 retval = Py_None;
 539                 break;
 540
 541         case TYPE_STOPITER:
 542                 Py_INCREF(PyExc_StopIteration);
 543                 retval = PyExc_StopIteration;
 544                 break;
 545
 546         case TYPE_ELLIPSIS:
 547                 Py_INCREF(Py_Ellipsis);
 548                 retval = Py_Ellipsis;
 549                 break;
 550
 551         case TYPE_FALSE:
 552                 Py_INCREF(Py_False);
 553                 retval = Py_False;
 554                 break;
 555
 556         case TYPE_TRUE:
 557                 Py_INCREF(Py_True);
 558                 retval = Py_True;
 559                 break;
 560
 561         case TYPE_INT:
 562                 retval = PyInt_FromLong(r_long(p));
 563                 break;
 564
 565         case TYPE_INT64:
 566                 retval = r_long64(p);
 567                 break;
 568
 569         case TYPE_LONG:
 570                 {
 571                         int size;
 572                         PyLongObject *ob;
 573                         n = r_long(p);
 574                         if (n < -INT_MAX || n > INT_MAX) {
 575                                 PyErr_SetString(PyExc_ValueError,
 576                                                 "bad marshal data");
 577                                 retval = NULL;
 578                                 break;
 579                         }
 580                         size = n<0 ? -n : n;
 581                         ob = _PyLong_New(size);
 582                         if (ob == NULL) {
 583                                 retval = NULL;
 584                                 break;
 585                         }
 586                         ob->ob_size = n;
 587                         for (i = 0; i < size; i++) {
 588                                 int digit = r_short(p);
 589                                 if (digit < 0) {
 590                                         Py_DECREF(ob);
 591                                         PyErr_SetString(PyExc_ValueError,
 592                                                         "bad marshal data");
 593                                         ob = NULL;
 594                                         break;
 595                                 }
 596                                 if (ob != NULL)
 597                                         ob->ob_digit[i] = digit;
 598                         }
 599                         retval = (PyObject *)ob;
 600                         break;
 601                 }
 602
 603         case TYPE_FLOAT:
 604                 {
 605                         char buf[256];
 606                         double dx;
 607                         n = r_byte(p);
 608                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 609                                 PyErr_SetString(PyExc_EOFError,
 610                                         "EOF read where object expected");
 611                                 retval = NULL;
 612                                 break;
 613                         }
 614                         buf[n] = '\0';
 615                         retval = NULL;
 616                         PyFPE_START_PROTECT("atof", break)
 617                         dx = PyOS_ascii_atof(buf);
 618                         PyFPE_END_PROTECT(dx)
 619                         retval = PyFloat_FromDouble(dx);
 620                         break;
 621                 }
 622
 623         case TYPE_BINARY_FLOAT:
 624                 {
 625                         unsigned char buf[8];
 626                         double x;
 627                         if (r_string((char*)buf, 8, p) != 8) {
 628                                 PyErr_SetString(PyExc_EOFError,
 629                                         "EOF read where object expected");
 630                                 retval = NULL;
 631                                 break;
 632                         }
 633                         x = _PyFloat_Unpack8(buf, 1);
 634                         if (x == -1.0 && PyErr_Occurred()) {
 635                                 retval = NULL;
 636                                 break;
 637                         }
 638                         retval = PyFloat_FromDouble(x);
 639                         break;
 640                 }
 641
 642 #ifndef WITHOUT_COMPLEX
 643         case TYPE_COMPLEX:
 644                 {
 645                         char buf[256];
 646                         Py_complex c;
 647                         n = r_byte(p);
 648                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 649                                 PyErr_SetString(PyExc_EOFError,
 650                                         "EOF read where object expected");
 651                                 retval = NULL;
 652                                 break;
 653                         }
 654                         buf[n] = '\0';
 655                         retval = NULL;
 656                         PyFPE_START_PROTECT("atof", break;)
 657                         c.real = PyOS_ascii_atof(buf);
 658                         PyFPE_END_PROTECT(c)
 659                         n = r_byte(p);
 660                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 661                                 PyErr_SetString(PyExc_EOFError,
 662                                         "EOF read where object expected");
 663                                 retval = NULL;
 664                                 break;
 665                         }
 666                         buf[n] = '\0';
 667                         PyFPE_START_PROTECT("atof", break)
 668                         c.imag = PyOS_ascii_atof(buf);
 669                         PyFPE_END_PROTECT(c)
 670                         retval = PyComplex_FromCComplex(c);
 671                         break;
 672                 }
 673
 674         case TYPE_BINARY_COMPLEX:
 675                 {
 676                         unsigned char buf[8];
 677                         Py_complex c;
 678                         if (r_string((char*)buf, 8, p) != 8) {
 679                                 PyErr_SetString(PyExc_EOFError,
 680                                         "EOF read where object expected");
 681                                 retval = NULL;
 682                                 break;
 683                         }
 684                         c.real = _PyFloat_Unpack8(buf, 1);
 685                         if (c.real == -1.0 && PyErr_Occurred()) {
 686                                 retval = NULL;
 687                                 break;
 688                         }
 689                         if (r_string((char*)buf, 8, p) != 8) {
 690                                 PyErr_SetString(PyExc_EOFError,
 691                                         "EOF read where object expected");
 692                                 retval = NULL;
 693                                 break;
 694                         }
 695                         c.imag = _PyFloat_Unpack8(buf, 1);
 696                         if (c.imag == -1.0 && PyErr_Occurred()) {
 697                                 retval = NULL;
 698                                 break;
 699                         }
 700                         retval = PyComplex_FromCComplex(c);
 701                         break;
 702                 }
 703 #endif
 704
 705         case TYPE_INTERNED:
 706         case TYPE_STRING:
 707                 n = r_long(p);
 708                 if (n < 0 || n > INT_MAX) {
 709                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 710                         retval = NULL;
 711                         break;
 712                 }
 713                 v = PyString_FromStringAndSize((char *)NULL, n);
 714                 if (v == NULL) {
 715                         retval = NULL;
 716                         break;
 717                 }
 718                 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
 719                         Py_DECREF(v);
 720                         PyErr_SetString(PyExc_EOFError,
 721                                         "EOF read where object expected");
 722                         retval = NULL;
 723                         break;
 724                 }
 725                 if (type == TYPE_INTERNED) {
 726                         PyString_InternInPlace(&v);
 727                         if (PyList_Append(p->strings, v) < 0) {
 728                                 retval = NULL;
 729                                 break;
 730                         }
 731                 }
 732                 retval = v;
 733                 break;
 734
 735         case TYPE_STRINGREF:
 736                 n = r_long(p);
 737                 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
 738                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 739                         retval = NULL;
 740                         break;
 741                 }
 742                 v = PyList_GET_ITEM(p->strings, n);
 743                 Py_INCREF(v);
 744                 retval = v;
 745                 break;
 746
 747 #ifdef Py_USING_UNICODE
 748         case TYPE_UNICODE:
 749             {
 750                 char *buffer;
 751
 752                 n = r_long(p);
 753                 if (n < 0 || n > INT_MAX) {
 754                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 755                         retval = NULL;
 756                         break;
 757                 }
 758                 buffer = PyMem_NEW(char, n);
 759                 if (buffer == NULL) {
 760                         retval = PyErr_NoMemory();
 761                         break;
 762                 }
 763                 if (r_string(buffer, (int)n, p) != n) {
 764                         PyMem_DEL(buffer);
 765                         PyErr_SetString(PyExc_EOFError,
 766                                 "EOF read where object expected");
 767                         retval = NULL;
 768                         break;
 769                 }
 770                 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
 771                 PyMem_DEL(buffer);
 772                 retval = v;
 773                 break;
 774             }
 775 #endif
 776
 777         case TYPE_TUPLE:
 778                 n = r_long(p);
 779                 if (n < 0 || n > INT_MAX) {
 780                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 781                         retval = NULL;
 782                         break;
 783                 }
 784                 v = PyTuple_New((int)n);
 785                 if (v == NULL) {
 786                         retval = NULL;
 787                         break;
 788                 }
 789                 for (i = 0; i < n; i++) {
 790                         v2 = r_object(p);
 791                         if ( v2 == NULL ) {
 792                                 if (!PyErr_Occurred())
 793                                         PyErr_SetString(PyExc_TypeError,
 794                                                 "NULL object in marshal data");
 795                                 Py_DECREF(v);
 796                                 v = NULL;
 797                                 break;
 798                         }
 799                         PyTuple_SET_ITEM(v, (int)i, v2);
 800                 }
 801                 retval = v;
 802                 break;
 803
 804         case TYPE_LIST:
 805                 n = r_long(p);
 806                 if (n < 0 || n > INT_MAX) {
 807                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 808                         retval = NULL;
 809                         break;
 810                 }
 811                 v = PyList_New((int)n);
 812                 if (v == NULL) {
 813                         retval = NULL;
 814                         break;
 815                 }
 816                 for (i = 0; i < n; i++) {
 817                         v2 = r_object(p);
 818                         if ( v2 == NULL ) {
 819                                 if (!PyErr_Occurred())
 820                                         PyErr_SetString(PyExc_TypeError,
 821                                                 "NULL object in marshal data");
 822                                 Py_DECREF(v);
 823                                 v = NULL;
 824                                 break;
 825                         }
 826                         PyList_SET_ITEM(v, (int)i, v2);
 827                 }
 828                 retval = v;
 829                 break;
 830
 831         case TYPE_DICT:
 832                 v = PyDict_New();
 833                 if (v == NULL) {
 834                         retval = NULL;
 835                         break;
 836                 }
 837                 for (;;) {
 838                         PyObject *key, *val;
 839                         key = r_object(p);
 840                         if (key == NULL)
 841                                 break;
 842                         val = r_object(p);
 843                         if (val != NULL)
 844                                 PyDict_SetItem(v, key, val);
 845                         Py_DECREF(key);
 846                         Py_XDECREF(val);
 847                 }
 848                 if (PyErr_Occurred()) {
 849                         Py_DECREF(v);
 850                         v = NULL;
 851                 }
 852                 retval = v;
 853                 break;
 854
 855         case TYPE_SET:
 856         case TYPE_FROZENSET:
 857                 n = r_long(p);
 858                 if (n < 0 || n > INT_MAX) {
 859                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 860                         retval = NULL;
 861                         break;
 862                 }
 863                 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
 864                 if (v == NULL) {
 865                         retval = NULL;
 866                         break;
 867                 }
 868                 for (i = 0; i < n; i++) {
 869                         v2 = r_object(p);
 870                         if ( v2 == NULL ) {
 871                                 if (!PyErr_Occurred())
 872                                         PyErr_SetString(PyExc_TypeError,
 873                                                 "NULL object in marshal data");
 874                                 Py_DECREF(v);
 875                                 v = NULL;
 876                                 break;
 877                         }
 878                         if (PySet_Add(v, v2) == -1) {
 879                                 Py_DECREF(v);
 880                                 Py_DECREF(v2);
 881                                 v = NULL;
 882                                 break;
 883                         }
 884                         Py_DECREF(v2);
 885                 }
 886                 retval = v;
 887                 break;
 888
 889         case TYPE_CODE:
 890                 if (PyEval_GetRestricted()) {
 891                         PyErr_SetString(PyExc_RuntimeError,
 892                                 "cannot unmarshal code objects in "
 893                                 "restricted execution mode");
 894                         retval = NULL;
 895                         break;
 896                 }
 897                 else {
 898                         int argcount;
 899                         int nlocals;
 900                         int stacksize;
 901                         int flags;
 902                         PyObject *code = NULL;
 903                         PyObject *consts = NULL;
 904                         PyObject *names = NULL;
 905                         PyObject *varnames = NULL;
 906                         PyObject *freevars = NULL;
 907                         PyObject *cellvars = NULL;
 908                         PyObject *filename = NULL;
 909                         PyObject *name = NULL;
 910                         int firstlineno;
 911                         PyObject *lnotab = NULL;
 912
 913                         v = NULL;
 914
 915                         /* XXX ignore long->int overflows for now */
 916                         argcount = (int)r_long(p);
 917                         nlocals = (int)r_long(p);
 918                         stacksize = (int)r_long(p);
 919                         flags = (int)r_long(p);
 920                         code = r_object(p);
 921                         if (code == NULL)
 922                                 goto code_error;
 923                         consts = r_object(p);
 924                         if (consts == NULL)
 925                                 goto code_error;
 926                         names = r_object(p);
 927                         if (names == NULL)
 928                                 goto code_error;
 929                         varnames = r_object(p);
 930                         if (varnames == NULL)
 931                                 goto code_error;
 932                         freevars = r_object(p);
 933                         if (freevars == NULL)
 934                                 goto code_error;
 935                         cellvars = r_object(p);
 936                         if (cellvars == NULL)
 937                                 goto code_error;
 938                         filename = r_object(p);
 939                         if (filename == NULL)
 940                                 goto code_error;
 941                         name = r_object(p);
 942                         if (name == NULL)
 943                                 goto code_error;
 944                         firstlineno = (int)r_long(p);
 945                         lnotab = r_object(p);
 946                         if (lnotab == NULL)
 947                                 goto code_error;
 948
 949                         v = (PyObject *) PyCode_New(
 950                                         argcount, nlocals, stacksize, flags,
 951                                         code, consts, names, varnames,
 952                                         freevars, cellvars, filename, name,
 953                                         firstlineno, lnotab);
 954
 955                   code_error:
 956                         Py_XDECREF(code);
 957                         Py_XDECREF(consts);
 958                         Py_XDECREF(names);
 959                         Py_XDECREF(varnames);
 960                         Py_XDECREF(freevars);
 961                         Py_XDECREF(cellvars);
 962                         Py_XDECREF(filename);
 963                         Py_XDECREF(name);
 964                         Py_XDECREF(lnotab);
 965
 966                 }
 967                 retval = v;
 968                 break;
 969
 970         default:
 971                 /* Bogus data got written, which isn't ideal.
 972                    This will let you keep working and recover. */
 973                 PyErr_SetString(PyExc_ValueError, "bad marshal data");
 974                 retval = NULL;
 975                 break;
 976
 977         }
 978         p->depth--;
 979         return retval;
 980 }
 981
 982 static PyObject *
 983 read_object(RFILE *p)
 984 {
 985         PyObject *v;
 986         if (PyErr_Occurred()) {
 987                 fprintf(stderr, "XXX readobject called with exception set\n");
 988                 return NULL;
 989         }
 990         v = r_object(p);
 991         if (v == NULL && !PyErr_Occurred())
 992                 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
 993         return v;
 994 }
 995
 996 int
 997 PyMarshal_ReadShortFromFile(FILE *fp)
 998 {
 999         RFILE rf;
1000         assert(fp);
1001         rf.fp = fp;
1002         rf.strings = NULL;
1003         rf.end = rf.ptr = NULL;
1004         return r_short(&rf);
1005 }
1006
1007 long
1008 PyMarshal_ReadLongFromFile(FILE *fp)
1009 {
1010         RFILE rf;
1011         rf.fp = fp;
1012         rf.strings = NULL;
1013         rf.ptr = rf.end = NULL;
1014         return r_long(&rf);
1015 }
1016
1017 #ifdef HAVE_FSTAT
1018 /* Return size of file in bytes; < 0 if unknown. */
1019 static off_t
1020 getfilesize(FILE *fp)
1021 {
1022         struct stat st;
1023         if (fstat(fileno(fp), &st) != 0)
1024                 return -1;
1025         else
1026                 return st.st_size;
1027 }
1028 #endif
1029
1030 /* If we can get the size of the file up-front, and it's reasonably small,
1031  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
1032  * than reading a byte at a time from file; speeds .pyc imports.
1033  * CAUTION:  since this may read the entire remainder of the file, don't
1034  * call it unless you know you're done with the file.
1035  */
1036 PyObject *
1037 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1038 {
1039 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1040  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1041  */
1042 #define SMALL_FILE_LIMIT (1L << 14)
1043 #define REASONABLE_FILE_LIMIT (1L << 18)
1044 #ifdef HAVE_FSTAT
1045         off_t filesize;
1046 #endif
1047 #ifdef HAVE_FSTAT
1048         filesize = getfilesize(fp);
1049         if (filesize > 0) {
1050                 char buf[SMALL_FILE_LIMIT];
1051                 char* pBuf = NULL;
1052                 if (filesize <= SMALL_FILE_LIMIT)
1053                         pBuf = buf;
1054                 else if (filesize <= REASONABLE_FILE_LIMIT)
1055                         pBuf = (char *)PyMem_MALLOC(filesize);
1056                 if (pBuf != NULL) {
1057                         PyObject* v;
1058                         size_t n;
1059                         /* filesize must fit into an int, because it
1060                            is smaller than REASONABLE_FILE_LIMIT */
1061                         n = fread(pBuf, 1, (int)filesize, fp);
1062                         v = PyMarshal_ReadObjectFromString(pBuf, n);
1063                         if (pBuf != buf)
1064                                 PyMem_FREE(pBuf);
1065                         return v;
1066                 }
1067
1068         }
1069 #endif
1070         /* We don't have fstat, or we do but the file is larger than
1071          * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1072          */
1073         return PyMarshal_ReadObjectFromFile(fp);
1074
1075 #undef SMALL_FILE_LIMIT
1076 #undef REASONABLE_FILE_LIMIT
1077 }
1078
1079 PyObject *
1080 PyMarshal_ReadObjectFromFile(FILE *fp)
1081 {
1082         RFILE rf;
1083         PyObject *result;
1084         rf.fp = fp;
1085         rf.strings = PyList_New(0);
1086         rf.depth = 0;
1087         rf.ptr = rf.end = NULL;
1088         result = r_object(&rf);
1089         Py_DECREF(rf.strings);
1090         return result;
1091 }
1092
1093 PyObject *
1094 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1095 {
1096         RFILE rf;
1097         PyObject *result;
1098         rf.fp = NULL;
1099         rf.ptr = str;
1100         rf.end = str + len;
1101         rf.strings = PyList_New(0);
1102         rf.depth = 0;
1103         result = r_object(&rf);
1104         Py_DECREF(rf.strings);
1105         return result;
1106 }
1107
1108 PyObject *
1109 PyMarshal_WriteObjectToString(PyObject *x, int version)
1110 {
1111         WFILE wf;
1112         wf.fp = NULL;
1113         wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1114         if (wf.str == NULL)
1115                 return NULL;
1116         wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1117         wf.end = wf.ptr + PyString_Size(wf.str);
1118         wf.error = 0;
1119         wf.depth = 0;
1120         wf.version = version;
1121         wf.strings = (version > 0) ? PyDict_New() : NULL;
1122         w_object(x, &wf);
1123         Py_XDECREF(wf.strings);
1124         if (wf.str != NULL) {
1125                 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1126                 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1127                         Py_DECREF(wf.str);
1128                         PyErr_SetString(PyExc_OverflowError,
1129                                         "too much marshall data for a string");
1130                         return NULL;
1131                 }
1132                 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1133         }
1134         if (wf.error) {
1135                 Py_XDECREF(wf.str);
1136                 PyErr_SetString(PyExc_ValueError,
1137                                 (wf.error==1)?"unmarshallable object"
1138                                 :"object too deeply nested to marshal");
1139                 return NULL;
1140         }
1141         return wf.str;
1142 }
1143
1144 /* And an interface for Python programs... */
1145
1146 static PyObject *
1147 marshal_dump(PyObject *self, PyObject *args)
1148 {
1149         WFILE wf;
1150         PyObject *x;
1151         PyObject *f;
1152         int version = Py_MARSHAL_VERSION;
1153         if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1154                 return NULL;
1155         if (!PyFile_Check(f)) {
1156                 PyErr_SetString(PyExc_TypeError,
1157                                 "marshal.dump() 2nd arg must be file");
1158                 return NULL;
1159         }
1160         wf.fp = PyFile_AsFile(f);
1161         wf.str = NULL;
1162         wf.ptr = wf.end = NULL;
1163         wf.error = 0;
1164         wf.depth = 0;
1165         wf.strings = (version > 0) ? PyDict_New() : 0;
1166         wf.version = version;
1167         w_object(x, &wf);
1168         Py_XDECREF(wf.strings);
1169         if (wf.error) {
1170                 PyErr_SetString(PyExc_ValueError,
1171                                 (wf.error==1)?"unmarshallable object"
1172                                 :"object too deeply nested to marshal");
1173                 return NULL;
1174         }
1175         Py_INCREF(Py_None);
1176         return Py_None;
1177 }
1178
1179 static PyObject *
1180 marshal_load(PyObject *self, PyObject *f)
1181 {
1182         RFILE rf;
1183         PyObject *result;
1184         if (!PyFile_Check(f)) {
1185                 PyErr_SetString(PyExc_TypeError,
1186                                 "marshal.load() arg must be file");
1187                 return NULL;
1188         }
1189         rf.fp = PyFile_AsFile(f);
1190         rf.strings = PyList_New(0);
1191         rf.depth = 0;
1192         result = read_object(&rf);
1193         Py_DECREF(rf.strings);
1194         return result;
1195 }
1196
1197 static PyObject *
1198 marshal_dumps(PyObject *self, PyObject *args)
1199 {
1200         PyObject *x;
1201         int version = Py_MARSHAL_VERSION;
1202         if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1203                 return NULL;
1204         return PyMarshal_WriteObjectToString(x, version);
1205 }
1206
1207 static PyObject *
1208 marshal_loads(PyObject *self, PyObject *args)
1209 {
1210         RFILE rf;
1211         char *s;
1212         Py_ssize_t n;
1213         PyObject* result;
1214         if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1215                 return NULL;
1216         rf.fp = NULL;
1217         rf.ptr = s;
1218         rf.end = s + n;
1219         rf.strings = PyList_New(0);
1220         rf.depth = 0;
1221         result = read_object(&rf);
1222         Py_DECREF(rf.strings);
1223         return result;
1224 }
1225
1226 static PyMethodDef marshal_methods[] = {
1227         {"dump",        marshal_dump,   METH_VARARGS},
1228         {"load",        marshal_load,   METH_O},
1229         {"dumps",       marshal_dumps,  METH_VARARGS},
1230         {"loads",       marshal_loads,  METH_VARARGS},
1231         {NULL,          NULL}           /* sentinel */
1232 };
1233
1234 PyMODINIT_FUNC
1235 PyMarshal_Init(void)
1236 {
1237         PyObject *mod = Py_InitModule("marshal", marshal_methods);
1238         if (mod == NULL)
1239                 return;
1240         PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1241 }