Python/marshal.c

   1
   2 /* Write Python objects to files and read them back.
   3    This is intended for writing and reading compiled Python code only;
   4    a true persistent storage facility would be much harder, since
   5    it would have to take circular links and sharing into account. */
   6
   7 #define PY_SSIZE_T_CLEAN
   8
   9 #include "Python.h"
  10 #include "longintrepr.h"
  11 #include "code.h"
  12 #include "marshal.h"
  13
  14 /* High water mark to determine when the marshalled object is dangerously deep
  15  * and risks coring the interpreter.  When the object stack gets this deep,
  16  * raise an exception instead of continuing.
  17  */
  18 #define MAX_MARSHAL_STACK_DEPTH 5000
  19
  20 #define TYPE_NULL               '0'
  21 #define TYPE_NONE               'N'
  22 #define TYPE_FALSE              'F'
  23 #define TYPE_TRUE               'T'
  24 #define TYPE_STOPITER           'S'
  25 #define TYPE_ELLIPSIS           '.'
  26 #define TYPE_INT                'i'
  27 #define TYPE_INT64              'I'
  28 #define TYPE_FLOAT              'f'
  29 #define TYPE_BINARY_FLOAT       'g'
  30 #define TYPE_COMPLEX            'x'
  31 #define TYPE_BINARY_COMPLEX     'y'
  32 #define TYPE_LONG               'l'
  33 #define TYPE_STRING             's'
  34 #define TYPE_INTERNED           't'
  35 #define TYPE_STRINGREF          'R'
  36 #define TYPE_TUPLE              '('
  37 #define TYPE_LIST               '['
  38 #define TYPE_DICT               '{'
  39 #define TYPE_CODE               'c'
  40 #define TYPE_UNICODE            'u'
  41 #define TYPE_UNKNOWN            '?'
  42 #define TYPE_SET                '<'
  43 #define TYPE_FROZENSET          '>'
  44
  45 typedef struct {
  46         FILE *fp;
  47         int error;
  48         int depth;
  49         /* If fp == NULL, the following are valid: */
  50         PyObject *str;
  51         char *ptr;
  52         char *end;
  53         PyObject *strings; /* dict on marshal, list on unmarshal */
  54         int version;
  55 } WFILE;
  56
  57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  58                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  59                            else w_more(c, p)
  60
  61 static void
  62 w_more(int c, WFILE *p)
  63 {
  64         Py_ssize_t size, newsize;
  65         if (p->str == NULL)
  66                 return; /* An error already occurred */
  67         size = PyString_Size(p->str);
  68         newsize = size + 1024;
  69         if (_PyString_Resize(&p->str, newsize) != 0) {
  70                 p->ptr = p->end = NULL;
  71         }
  72         else {
  73                 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
  74                 p->end =
  75                         PyString_AS_STRING((PyStringObject *)p->str) + newsize;
  76                 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
  77         }
  78 }
  79
  80 static void
  81 w_string(char *s, int n, WFILE *p)
  82 {
  83         if (p->fp != NULL) {
  84                 fwrite(s, 1, n, p->fp);
  85         }
  86         else {
  87                 while (--n >= 0) {
  88                         w_byte(*s, p);
  89                         s++;
  90                 }
  91         }
  92 }
  93
  94 static void
  95 w_short(int x, WFILE *p)
  96 {
  97         w_byte((char)( x      & 0xff), p);
  98         w_byte((char)((x>> 8) & 0xff), p);
  99 }
 100
 101 static void
 102 w_long(long x, WFILE *p)
 103 {
 104         w_byte((char)( x      & 0xff), p);
 105         w_byte((char)((x>> 8) & 0xff), p);
 106         w_byte((char)((x>>16) & 0xff), p);
 107         w_byte((char)((x>>24) & 0xff), p);
 108 }
 109
 110 #if SIZEOF_LONG > 4
 111 static void
 112 w_long64(long x, WFILE *p)
 113 {
 114         w_long(x, p);
 115         w_long(x>>32, p);
 116 }
 117 #endif
 118
 119 static void
 120 w_object(PyObject *v, WFILE *p)
 121 {
 122         Py_ssize_t i, n;
 123
 124         p->depth++;
 125
 126         if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 127                 p->error = 2;
 128         }
 129         else if (v == NULL) {
 130                 w_byte(TYPE_NULL, p);
 131         }
 132         else if (v == Py_None) {
 133                 w_byte(TYPE_NONE, p);
 134         }
 135         else if (v == PyExc_StopIteration) {
 136                 w_byte(TYPE_STOPITER, p);
 137         }
 138         else if (v == Py_Ellipsis) {
 139                 w_byte(TYPE_ELLIPSIS, p);
 140         }
 141         else if (v == Py_False) {
 142                 w_byte(TYPE_FALSE, p);
 143         }
 144         else if (v == Py_True) {
 145                 w_byte(TYPE_TRUE, p);
 146         }
 147         else if (PyInt_Check(v)) {
 148                 long x = PyInt_AS_LONG((PyIntObject *)v);
 149 #if SIZEOF_LONG > 4
 150                 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
 151                 if (y && y != -1) {
 152                         w_byte(TYPE_INT64, p);
 153                         w_long64(x, p);
 154                 }
 155                 else
 156 #endif
 157                         {
 158                         w_byte(TYPE_INT, p);
 159                         w_long(x, p);
 160                 }
 161         }
 162         else if (PyLong_Check(v)) {
 163                 PyLongObject *ob = (PyLongObject *)v;
 164                 w_byte(TYPE_LONG, p);
 165                 n = ob->ob_size;
 166                 w_long((long)n, p);
 167                 if (n < 0)
 168                         n = -n;
 169                 for (i = 0; i < n; i++)
 170                         w_short(ob->ob_digit[i], p);
 171         }
 172         else if (PyFloat_Check(v)) {
 173                 if (p->version > 1) {
 174                         unsigned char buf[8];
 175                         if (_PyFloat_Pack8(PyFloat_AsDouble(v),
 176                                            buf, 1) < 0) {
 177                                 p->error = 1;
 178                                 return;
 179                         }
 180                         w_byte(TYPE_BINARY_FLOAT, p);
 181                         w_string((char*)buf, 8, p);
 182                 }
 183                 else {
 184                         char buf[256]; /* Plenty to format any double */
 185                         PyFloat_AsReprString(buf, (PyFloatObject *)v);
 186                         n = strlen(buf);
 187                         w_byte(TYPE_FLOAT, p);
 188                         w_byte((int)n, p);
 189                         w_string(buf, (int)n, p);
 190                 }
 191         }
 192 #ifndef WITHOUT_COMPLEX
 193         else if (PyComplex_Check(v)) {
 194                 if (p->version > 1) {
 195                         unsigned char buf[8];
 196                         if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
 197                                            buf, 1) < 0) {
 198                                 p->error = 1;
 199                                 return;
 200                         }
 201                         w_byte(TYPE_BINARY_COMPLEX, p);
 202                         w_string((char*)buf, 8, p);
 203                         if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
 204                                            buf, 1) < 0) {
 205                                 p->error = 1;
 206                                 return;
 207                         }
 208                         w_string((char*)buf, 8, p);
 209                 }
 210                 else {
 211                         char buf[256]; /* Plenty to format any double */
 212                         PyFloatObject *temp;
 213                         w_byte(TYPE_COMPLEX, p);
 214                         temp = (PyFloatObject*)PyFloat_FromDouble(
 215                                 PyComplex_RealAsDouble(v));
 216                         PyFloat_AsReprString(buf, temp);
 217                         Py_DECREF(temp);
 218                         n = strlen(buf);
 219                         w_byte((int)n, p);
 220                         w_string(buf, (int)n, p);
 221                         temp = (PyFloatObject*)PyFloat_FromDouble(
 222                                 PyComplex_ImagAsDouble(v));
 223                         PyFloat_AsReprString(buf, temp);
 224                         Py_DECREF(temp);
 225                         n = strlen(buf);
 226                         w_byte((int)n, p);
 227                         w_string(buf, (int)n, p);
 228                 }
 229         }
 230 #endif
 231         else if (PyString_Check(v)) {
 232                 if (p->strings && PyString_CHECK_INTERNED(v)) {
 233                         PyObject *o = PyDict_GetItem(p->strings, v);
 234                         if (o) {
 235                                 long w = PyInt_AsLong(o);
 236                                 w_byte(TYPE_STRINGREF, p);
 237                                 w_long(w, p);
 238                                 goto exit;
 239                         }
 240                         else {
 241                                 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
 242                                 PyDict_SetItem(p->strings, v, o);
 243                                 Py_DECREF(o);
 244                                 w_byte(TYPE_INTERNED, p);
 245                         }
 246                 }
 247                 else {
 248                         w_byte(TYPE_STRING, p);
 249                 }
 250                 n = PyString_GET_SIZE(v);
 251                 if (n > INT_MAX) {
 252                         /* huge strings are not supported */
 253                         p->depth--;
 254                         p->error = 1;
 255                         return;
 256                 }
 257                 w_long((long)n, p);
 258                 w_string(PyString_AS_STRING(v), (int)n, p);
 259         }
 260 #ifdef Py_USING_UNICODE
 261         else if (PyUnicode_Check(v)) {
 262                 PyObject *utf8;
 263                 utf8 = PyUnicode_AsUTF8String(v);
 264                 if (utf8 == NULL) {
 265                         p->depth--;
 266                         p->error = 1;
 267                         return;
 268                 }
 269                 w_byte(TYPE_UNICODE, p);
 270                 n = PyString_GET_SIZE(utf8);
 271                 if (n > INT_MAX) {
 272                         p->depth--;
 273                         p->error = 1;
 274                         return;
 275                 }
 276                 w_long((long)n, p);
 277                 w_string(PyString_AS_STRING(utf8), (int)n, p);
 278                 Py_DECREF(utf8);
 279         }
 280 #endif
 281         else if (PyTuple_Check(v)) {
 282                 w_byte(TYPE_TUPLE, p);
 283                 n = PyTuple_Size(v);
 284                 w_long((long)n, p);
 285                 for (i = 0; i < n; i++) {
 286                         w_object(PyTuple_GET_ITEM(v, i), p);
 287                 }
 288         }
 289         else if (PyList_Check(v)) {
 290                 w_byte(TYPE_LIST, p);
 291                 n = PyList_GET_SIZE(v);
 292                 w_long((long)n, p);
 293                 for (i = 0; i < n; i++) {
 294                         w_object(PyList_GET_ITEM(v, i), p);
 295                 }
 296         }
 297         else if (PyDict_Check(v)) {
 298                 Py_ssize_t pos;
 299                 PyObject *key, *value;
 300                 w_byte(TYPE_DICT, p);
 301                 /* This one is NULL object terminated! */
 302                 pos = 0;
 303                 while (PyDict_Next(v, &pos, &key, &value)) {
 304                         w_object(key, p);
 305                         w_object(value, p);
 306                 }
 307                 w_object((PyObject *)NULL, p);
 308         }
 309         else if (PyAnySet_Check(v)) {
 310                 PyObject *value, *it;
 311
 312                 if (PyObject_TypeCheck(v, &PySet_Type))
 313                         w_byte(TYPE_SET, p);
 314                 else
 315                         w_byte(TYPE_FROZENSET, p);
 316                 n = PyObject_Size(v);
 317                 if (n == -1) {
 318                         p->depth--;
 319                         p->error = 1;
 320                         return;
 321                 }
 322                 w_long((long)n, p);
 323                 it = PyObject_GetIter(v);
 324                 if (it == NULL) {
 325                         p->depth--;
 326                         p->error = 1;
 327                         return;
 328                 }
 329                 while ((value = PyIter_Next(it)) != NULL) {
 330                         w_object(value, p);
 331                         Py_DECREF(value);
 332                 }
 333                 Py_DECREF(it);
 334                 if (PyErr_Occurred()) {
 335                         p->depth--;
 336                         p->error = 1;
 337                         return;
 338                 }
 339         }
 340         else if (PyCode_Check(v)) {
 341                 PyCodeObject *co = (PyCodeObject *)v;
 342                 w_byte(TYPE_CODE, p);
 343                 w_long(co->co_argcount, p);
 344                 w_long(co->co_nlocals, p);
 345                 w_long(co->co_stacksize, p);
 346                 w_long(co->co_flags, p);
 347                 w_object(co->co_code, p);
 348                 w_object(co->co_consts, p);
 349                 w_object(co->co_names, p);
 350                 w_object(co->co_varnames, p);
 351                 w_object(co->co_freevars, p);
 352                 w_object(co->co_cellvars, p);
 353                 w_object(co->co_filename, p);
 354                 w_object(co->co_name, p);
 355                 w_long(co->co_firstlineno, p);
 356                 w_object(co->co_lnotab, p);
 357         }
 358         else if (PyObject_CheckReadBuffer(v)) {
 359                 /* Write unknown buffer-style objects as a string */
 360                 char *s;
 361                 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
 362                 w_byte(TYPE_STRING, p);
 363                 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
 364                 if (n > INT_MAX) {
 365                         p->depth--;
 366                         p->error = 1;
 367                         return;
 368                 }
 369                 w_long((long)n, p);
 370                 w_string(s, (int)n, p);
 371         }
 372         else {
 373                 w_byte(TYPE_UNKNOWN, p);
 374                 p->error = 1;
 375         }
 376    exit:
 377         p->depth--;
 378 }
 379
 380 /* version currently has no effect for writing longs. */
 381 void
 382 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 383 {
 384         WFILE wf;
 385         wf.fp = fp;
 386         wf.error = 0;
 387         wf.depth = 0;
 388         wf.strings = NULL;
 389         wf.version = version;
 390         w_long(x, &wf);
 391 }
 392
 393 void
 394 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 395 {
 396         WFILE wf;
 397         wf.fp = fp;
 398         wf.error = 0;
 399         wf.depth = 0;
 400         wf.strings = (version > 0) ? PyDict_New() : NULL;
 401         wf.version = version;
 402         w_object(x, &wf);
 403         Py_XDECREF(wf.strings);
 404 }
 405
 406 typedef WFILE RFILE; /* Same struct with different invariants */
 407
 408 #define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 409
 410 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 411
 412 static int
 413 r_string(char *s, int n, RFILE *p)
 414 {
 415         if (p->fp != NULL)
 416                 /* The result fits into int because it must be <=n. */
 417                 return (int)fread(s, 1, n, p->fp);
 418         if (p->end - p->ptr < n)
 419                 n = (int)(p->end - p->ptr);
 420         memcpy(s, p->ptr, n);
 421         p->ptr += n;
 422         return n;
 423 }
 424
 425 static int
 426 r_short(RFILE *p)
 427 {
 428         register short x;
 429         x = r_byte(p);
 430         x |= r_byte(p) << 8;
 431         /* Sign-extension, in case short greater than 16 bits */
 432         x |= -(x & 0x8000);
 433         return x;
 434 }
 435
 436 static long
 437 r_long(RFILE *p)
 438 {
 439         register long x;
 440         register FILE *fp = p->fp;
 441         if (fp) {
 442                 x = getc(fp);
 443                 x |= (long)getc(fp) << 8;
 444                 x |= (long)getc(fp) << 16;
 445                 x |= (long)getc(fp) << 24;
 446         }
 447         else {
 448                 x = rs_byte(p);
 449                 x |= (long)rs_byte(p) << 8;
 450                 x |= (long)rs_byte(p) << 16;
 451                 x |= (long)rs_byte(p) << 24;
 452         }
 453 #if SIZEOF_LONG > 4
 454         /* Sign extension for 64-bit machines */
 455         x |= -(x & 0x80000000L);
 456 #endif
 457         return x;
 458 }
 459
 460 /* r_long64 deals with the TYPE_INT64 code.  On a machine with
 461    sizeof(long) > 4, it returns a Python int object, else a Python long
 462    object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
 463    so there's no inefficiency here in returning a PyLong on 32-bit boxes
 464    for everything written via TYPE_INT64 (i.e., if an int is written via
 465    TYPE_INT64, it *needs* more than 32 bits).
 466 */
 467 static PyObject *
 468 r_long64(RFILE *p)
 469 {
 470         long lo4 = r_long(p);
 471         long hi4 = r_long(p);
 472 #if SIZEOF_LONG > 4
 473         long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
 474         return PyInt_FromLong(x);
 475 #else
 476         unsigned char buf[8];
 477         int one = 1;
 478         int is_little_endian = (int)*(char*)&one;
 479         if (is_little_endian) {
 480                 memcpy(buf, &lo4, 4);
 481                 memcpy(buf+4, &hi4, 4);
 482         }
 483         else {
 484                 memcpy(buf, &hi4, 4);
 485                 memcpy(buf+4, &lo4, 4);
 486         }
 487         return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 488 #endif
 489 }
 490
 491 static PyObject *
 492 r_object(RFILE *p)
 493 {
 494         /* NULL is a valid return value, it does not necessarily means that
 495            an exception is set. */
 496         PyObject *v, *v2, *v3;
 497         long i, n;
 498         int type = r_byte(p);
 499
 500         switch (type) {
 501
 502         case EOF:
 503                 PyErr_SetString(PyExc_EOFError,
 504                                 "EOF read where object expected");
 505                 return NULL;
 506
 507         case TYPE_NULL:
 508                 return NULL;
 509
 510         case TYPE_NONE:
 511                 Py_INCREF(Py_None);
 512                 return Py_None;
 513
 514         case TYPE_STOPITER:
 515                 Py_INCREF(PyExc_StopIteration);
 516                 return PyExc_StopIteration;
 517
 518         case TYPE_ELLIPSIS:
 519                 Py_INCREF(Py_Ellipsis);
 520                 return Py_Ellipsis;
 521
 522         case TYPE_FALSE:
 523                 Py_INCREF(Py_False);
 524                 return Py_False;
 525
 526         case TYPE_TRUE:
 527                 Py_INCREF(Py_True);
 528                 return Py_True;
 529
 530         case TYPE_INT:
 531                 return PyInt_FromLong(r_long(p));
 532
 533         case TYPE_INT64:
 534                 return r_long64(p);
 535
 536         case TYPE_LONG:
 537                 {
 538                         int size;
 539                         PyLongObject *ob;
 540                         n = r_long(p);
 541                         size = n<0 ? -n : n;
 542                         ob = _PyLong_New(size);
 543                         if (ob == NULL)
 544                                 return NULL;
 545                         ob->ob_size = n;
 546                         for (i = 0; i < size; i++) {
 547                                 int digit = r_short(p);
 548                                 if (digit < 0) {
 549                                         Py_DECREF(ob);
 550                                         PyErr_SetString(PyExc_ValueError,
 551                                                         "bad marshal data");
 552                                         return NULL;
 553                                 }
 554                                 ob->ob_digit[i] = digit;
 555                         }
 556                         return (PyObject *)ob;
 557                 }
 558
 559         case TYPE_FLOAT:
 560                 {
 561                         char buf[256];
 562                         double dx;
 563                         n = r_byte(p);
 564                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 565                                 PyErr_SetString(PyExc_EOFError,
 566                                         "EOF read where object expected");
 567                                 return NULL;
 568                         }
 569                         buf[n] = '\0';
 570                         PyFPE_START_PROTECT("atof", return 0)
 571                         dx = PyOS_ascii_atof(buf);
 572                         PyFPE_END_PROTECT(dx)
 573                         return PyFloat_FromDouble(dx);
 574                 }
 575
 576         case TYPE_BINARY_FLOAT:
 577                 {
 578                         unsigned char buf[8];
 579                         double x;
 580                         if (r_string((char*)buf, 8, p) != 8) {
 581                                 PyErr_SetString(PyExc_EOFError,
 582                                         "EOF read where object expected");
 583                                 return NULL;
 584                         }
 585                         x = _PyFloat_Unpack8(buf, 1);
 586                         if (x == -1.0 && PyErr_Occurred()) {
 587                                 return NULL;
 588                         }
 589                         return PyFloat_FromDouble(x);
 590                 }
 591
 592 #ifndef WITHOUT_COMPLEX
 593         case TYPE_COMPLEX:
 594                 {
 595                         char buf[256];
 596                         Py_complex c;
 597                         n = r_byte(p);
 598                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 599                                 PyErr_SetString(PyExc_EOFError,
 600                                         "EOF read where object expected");
 601                                 return NULL;
 602                         }
 603                         buf[n] = '\0';
 604                         PyFPE_START_PROTECT("atof", return 0)
 605                         c.real = PyOS_ascii_atof(buf);
 606                         PyFPE_END_PROTECT(c)
 607                         n = r_byte(p);
 608                         if (n == EOF || r_string(buf, (int)n, p) != n) {
 609                                 PyErr_SetString(PyExc_EOFError,
 610                                         "EOF read where object expected");
 611                                 return NULL;
 612                         }
 613                         buf[n] = '\0';
 614                         PyFPE_START_PROTECT("atof", return 0)
 615                         c.imag = PyOS_ascii_atof(buf);
 616                         PyFPE_END_PROTECT(c)
 617                         return PyComplex_FromCComplex(c);
 618                 }
 619
 620         case TYPE_BINARY_COMPLEX:
 621                 {
 622                         unsigned char buf[8];
 623                         Py_complex c;
 624                         if (r_string((char*)buf, 8, p) != 8) {
 625                                 PyErr_SetString(PyExc_EOFError,
 626                                         "EOF read where object expected");
 627                                 return NULL;
 628                         }
 629                         c.real = _PyFloat_Unpack8(buf, 1);
 630                         if (c.real == -1.0 && PyErr_Occurred()) {
 631                                 return NULL;
 632                         }
 633                         if (r_string((char*)buf, 8, p) != 8) {
 634                                 PyErr_SetString(PyExc_EOFError,
 635                                         "EOF read where object expected");
 636                                 return NULL;
 637                         }
 638                         c.imag = _PyFloat_Unpack8(buf, 1);
 639                         if (c.imag == -1.0 && PyErr_Occurred()) {
 640                                 return NULL;
 641                         }
 642                         return PyComplex_FromCComplex(c);
 643                 }
 644 #endif
 645
 646         case TYPE_INTERNED:
 647         case TYPE_STRING:
 648                 n = r_long(p);
 649                 if (n < 0) {
 650                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 651                         return NULL;
 652                 }
 653                 v = PyString_FromStringAndSize((char *)NULL, n);
 654                 if (v == NULL)
 655                         return v;
 656                 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
 657                         Py_DECREF(v);
 658                         PyErr_SetString(PyExc_EOFError,
 659                                         "EOF read where object expected");
 660                         return NULL;
 661                 }
 662                 if (type == TYPE_INTERNED) {
 663                         PyString_InternInPlace(&v);
 664                         PyList_Append(p->strings, v);
 665                 }
 666                 return v;
 667
 668         case TYPE_STRINGREF:
 669                 n = r_long(p);
 670                 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
 671                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 672                         return NULL;
 673                 }
 674                 v = PyList_GET_ITEM(p->strings, n);
 675                 Py_INCREF(v);
 676                 return v;
 677
 678 #ifdef Py_USING_UNICODE
 679         case TYPE_UNICODE:
 680             {
 681                 char *buffer;
 682
 683                 n = r_long(p);
 684                 if (n < 0) {
 685                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 686                         return NULL;
 687                 }
 688                 buffer = PyMem_NEW(char, n);
 689                 if (buffer == NULL)
 690                         return PyErr_NoMemory();
 691                 if (r_string(buffer, (int)n, p) != n) {
 692                         PyMem_DEL(buffer);
 693                         PyErr_SetString(PyExc_EOFError,
 694                                 "EOF read where object expected");
 695                         return NULL;
 696                 }
 697                 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
 698                 PyMem_DEL(buffer);
 699                 return v;
 700             }
 701 #endif
 702
 703         case TYPE_TUPLE:
 704                 n = r_long(p);
 705                 if (n < 0) {
 706                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 707                         return NULL;
 708                 }
 709                 v = PyTuple_New((int)n);
 710                 if (v == NULL)
 711                         return v;
 712                 for (i = 0; i < n; i++) {
 713                         v2 = r_object(p);
 714                         if ( v2 == NULL ) {
 715                                 if (!PyErr_Occurred())
 716                                         PyErr_SetString(PyExc_TypeError,
 717                                                 "NULL object in marshal data");
 718                                 Py_DECREF(v);
 719                                 v = NULL;
 720                                 break;
 721                         }
 722                         PyTuple_SET_ITEM(v, (int)i, v2);
 723                 }
 724                 return v;
 725
 726         case TYPE_LIST:
 727                 n = r_long(p);
 728                 if (n < 0) {
 729                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 730                         return NULL;
 731                 }
 732                 v = PyList_New((int)n);
 733                 if (v == NULL)
 734                         return v;
 735                 for (i = 0; i < n; i++) {
 736                         v2 = r_object(p);
 737                         if ( v2 == NULL ) {
 738                                 if (!PyErr_Occurred())
 739                                         PyErr_SetString(PyExc_TypeError,
 740                                                 "NULL object in marshal data");
 741                                 Py_DECREF(v);
 742                                 v = NULL;
 743                                 break;
 744                         }
 745                         PyList_SetItem(v, (int)i, v2);
 746                 }
 747                 return v;
 748
 749         case TYPE_DICT:
 750                 v = PyDict_New();
 751                 if (v == NULL)
 752                         return NULL;
 753                 for (;;) {
 754                         PyObject *key, *val;
 755                         key = r_object(p);
 756                         if (key == NULL)
 757                                 break;
 758                         val = r_object(p);
 759                         if (val != NULL)
 760                                 PyDict_SetItem(v, key, val);
 761                         Py_DECREF(key);
 762                         Py_XDECREF(val);
 763                 }
 764                 if (PyErr_Occurred()) {
 765                         Py_DECREF(v);
 766                         v = NULL;
 767                 }
 768                 return v;
 769
 770         case TYPE_SET:
 771         case TYPE_FROZENSET:
 772                 n = r_long(p);
 773                 if (n < 0) {
 774                         PyErr_SetString(PyExc_ValueError, "bad marshal data");
 775                         return NULL;
 776                 }
 777                 v = PyTuple_New((int)n);
 778                 if (v == NULL)
 779                         return v;
 780                 for (i = 0; i < n; i++) {
 781                         v2 = r_object(p);
 782                         if ( v2 == NULL ) {
 783                                 if (!PyErr_Occurred())
 784                                         PyErr_SetString(PyExc_TypeError,
 785                                                 "NULL object in marshal data");
 786                                 Py_DECREF(v);
 787                                 v = NULL;
 788                                 break;
 789                         }
 790                         PyTuple_SET_ITEM(v, (int)i, v2);
 791                 }
 792                 if (v == NULL)
 793                         return v;
 794                 if (type == TYPE_SET)
 795                         v3 = PySet_New(v);
 796                 else
 797                         v3 = PyFrozenSet_New(v);
 798                 Py_DECREF(v);
 799                 return v3;
 800
 801         case TYPE_CODE:
 802                 if (PyEval_GetRestricted()) {
 803                         PyErr_SetString(PyExc_RuntimeError,
 804                                 "cannot unmarshal code objects in "
 805                                 "restricted execution mode");
 806                         return NULL;
 807                 }
 808                 else {
 809                         int argcount;
 810                         int nlocals;
 811                         int stacksize;
 812                         int flags;
 813                         PyObject *code = NULL;
 814                         PyObject *consts = NULL;
 815                         PyObject *names = NULL;
 816                         PyObject *varnames = NULL;
 817                         PyObject *freevars = NULL;
 818                         PyObject *cellvars = NULL;
 819                         PyObject *filename = NULL;
 820                         PyObject *name = NULL;
 821                         int firstlineno;
 822                         PyObject *lnotab = NULL;
 823
 824                         v = NULL;
 825
 826                         argcount = r_long(p);
 827                         nlocals = r_long(p);
 828                         stacksize = r_long(p);
 829                         flags = r_long(p);
 830                         code = r_object(p);
 831                         if (code == NULL)
 832                                 goto code_error;
 833                         consts = r_object(p);
 834                         if (consts == NULL)
 835                                 goto code_error;
 836                         names = r_object(p);
 837                         if (names == NULL)
 838                                 goto code_error;
 839                         varnames = r_object(p);
 840                         if (varnames == NULL)
 841                                 goto code_error;
 842                         freevars = r_object(p);
 843                         if (freevars == NULL)
 844                                 goto code_error;
 845                         cellvars = r_object(p);
 846                         if (cellvars == NULL)
 847                                 goto code_error;
 848                         filename = r_object(p);
 849                         if (filename == NULL)
 850                                 goto code_error;
 851                         name = r_object(p);
 852                         if (name == NULL)
 853                                 goto code_error;
 854                         firstlineno = r_long(p);
 855                         lnotab = r_object(p);
 856                         if (lnotab == NULL)
 857                                 goto code_error;
 858
 859                         v = (PyObject *) PyCode_New(
 860                                         argcount, nlocals, stacksize, flags,
 861                                         code, consts, names, varnames,
 862                                         freevars, cellvars, filename, name,
 863                                         firstlineno, lnotab);
 864
 865                   code_error:
 866                         Py_XDECREF(code);
 867                         Py_XDECREF(consts);
 868                         Py_XDECREF(names);
 869                         Py_XDECREF(varnames);
 870                         Py_XDECREF(freevars);
 871                         Py_XDECREF(cellvars);
 872                         Py_XDECREF(filename);
 873                         Py_XDECREF(name);
 874                         Py_XDECREF(lnotab);
 875
 876                 }
 877                 return v;
 878
 879         default:
 880                 /* Bogus data got written, which isn't ideal.
 881                    This will let you keep working and recover. */
 882                 PyErr_SetString(PyExc_ValueError, "bad marshal data");
 883                 return NULL;
 884
 885         }
 886 }
 887
 888 static PyObject *
 889 read_object(RFILE *p)
 890 {
 891         PyObject *v;
 892         if (PyErr_Occurred()) {
 893                 fprintf(stderr, "XXX readobject called with exception set\n");
 894                 return NULL;
 895         }
 896         v = r_object(p);
 897         if (v == NULL && !PyErr_Occurred())
 898                 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
 899         return v;
 900 }
 901
 902 int
 903 PyMarshal_ReadShortFromFile(FILE *fp)
 904 {
 905         RFILE rf;
 906         assert(fp);
 907         rf.fp = fp;
 908         rf.strings = NULL;
 909         rf.end = rf.ptr = NULL;
 910         return r_short(&rf);
 911 }
 912
 913 long
 914 PyMarshal_ReadLongFromFile(FILE *fp)
 915 {
 916         RFILE rf;
 917         rf.fp = fp;
 918         rf.strings = NULL;
 919         return r_long(&rf);
 920 }
 921
 922 #ifdef HAVE_FSTAT
 923 /* Return size of file in bytes; < 0 if unknown. */
 924 static off_t
 925 getfilesize(FILE *fp)
 926 {
 927         struct stat st;
 928         if (fstat(fileno(fp), &st) != 0)
 929                 return -1;
 930         else
 931                 return st.st_size;
 932 }
 933 #endif
 934
 935 /* If we can get the size of the file up-front, and it's reasonably small,
 936  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
 937  * than reading a byte at a time from file; speeds .pyc imports.
 938  * CAUTION:  since this may read the entire remainder of the file, don't
 939  * call it unless you know you're done with the file.
 940  */
 941 PyObject *
 942 PyMarshal_ReadLastObjectFromFile(FILE *fp)
 943 {
 944 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
 945  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
 946  */
 947 #define SMALL_FILE_LIMIT (1L << 14)
 948 #define REASONABLE_FILE_LIMIT (1L << 18)
 949 #ifdef HAVE_FSTAT
 950         off_t filesize;
 951 #endif
 952 #ifdef HAVE_FSTAT
 953         filesize = getfilesize(fp);
 954         if (filesize > 0) {
 955                 char buf[SMALL_FILE_LIMIT];
 956                 char* pBuf = NULL;
 957                 if (filesize <= SMALL_FILE_LIMIT)
 958                         pBuf = buf;
 959                 else if (filesize <= REASONABLE_FILE_LIMIT)
 960                         pBuf = (char *)PyMem_MALLOC(filesize);
 961                 if (pBuf != NULL) {
 962                         PyObject* v;
 963                         size_t n;
 964                         /* filesize must fit into an int, because it
 965                            is smaller than REASONABLE_FILE_LIMIT */
 966                         n = fread(pBuf, 1, (int)filesize, fp);
 967                         v = PyMarshal_ReadObjectFromString(pBuf, n);
 968                         if (pBuf != buf)
 969                                 PyMem_FREE(pBuf);
 970                         return v;
 971                 }
 972
 973         }
 974 #endif
 975         /* We don't have fstat, or we do but the file is larger than
 976          * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
 977          */
 978         return PyMarshal_ReadObjectFromFile(fp);
 979
 980 #undef SMALL_FILE_LIMIT
 981 #undef REASONABLE_FILE_LIMIT
 982 }
 983
 984 PyObject *
 985 PyMarshal_ReadObjectFromFile(FILE *fp)
 986 {
 987         RFILE rf;
 988         PyObject *result;
 989         rf.fp = fp;
 990         rf.strings = PyList_New(0);
 991         result = r_object(&rf);
 992         Py_DECREF(rf.strings);
 993         return result;
 994 }
 995
 996 PyObject *
 997 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
 998 {
 999         RFILE rf;
1000         PyObject *result;
1001         rf.fp = NULL;
1002         rf.ptr = str;
1003         rf.end = str + len;
1004         rf.strings = PyList_New(0);
1005         result = r_object(&rf);
1006         Py_DECREF(rf.strings);
1007         return result;
1008 }
1009
1010 PyObject *
1011 PyMarshal_WriteObjectToString(PyObject *x, int version)
1012 {
1013         WFILE wf;
1014         wf.fp = NULL;
1015         wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1016         if (wf.str == NULL)
1017                 return NULL;
1018         wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1019         wf.end = wf.ptr + PyString_Size(wf.str);
1020         wf.error = 0;
1021         wf.depth = 0;
1022         wf.version = version;
1023         wf.strings = (version > 0) ? PyDict_New() : NULL;
1024         w_object(x, &wf);
1025         Py_XDECREF(wf.strings);
1026         if (wf.str != NULL)
1027                 _PyString_Resize(&wf.str,
1028                     (int) (wf.ptr -
1029                            PyString_AS_STRING((PyStringObject *)wf.str)));
1030         if (wf.error) {
1031                 Py_XDECREF(wf.str);
1032                 PyErr_SetString(PyExc_ValueError,
1033                                 (wf.error==1)?"unmarshallable object"
1034                                 :"object too deeply nested to marshal");
1035                 return NULL;
1036         }
1037         return wf.str;
1038 }
1039
1040 /* And an interface for Python programs... */
1041
1042 static PyObject *
1043 marshal_dump(PyObject *self, PyObject *args)
1044 {
1045         WFILE wf;
1046         PyObject *x;
1047         PyObject *f;
1048         int version = Py_MARSHAL_VERSION;
1049         if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1050                 return NULL;
1051         if (!PyFile_Check(f)) {
1052                 PyErr_SetString(PyExc_TypeError,
1053                                 "marshal.dump() 2nd arg must be file");
1054                 return NULL;
1055         }
1056         wf.fp = PyFile_AsFile(f);
1057         wf.str = NULL;
1058         wf.ptr = wf.end = NULL;
1059         wf.error = 0;
1060         wf.depth = 0;
1061         wf.strings = (version > 0) ? PyDict_New() : 0;
1062         wf.version = version;
1063         w_object(x, &wf);
1064         Py_XDECREF(wf.strings);
1065         if (wf.error) {
1066                 PyErr_SetString(PyExc_ValueError,
1067                                 (wf.error==1)?"unmarshallable object"
1068                                 :"object too deeply nested to marshal");
1069                 return NULL;
1070         }
1071         Py_INCREF(Py_None);
1072         return Py_None;
1073 }
1074
1075 static PyObject *
1076 marshal_load(PyObject *self, PyObject *args)
1077 {
1078         RFILE rf;
1079         PyObject *f, *result;
1080         if (!PyArg_ParseTuple(args, "O:load", &f))
1081                 return NULL;
1082         if (!PyFile_Check(f)) {
1083                 PyErr_SetString(PyExc_TypeError,
1084                                 "marshal.load() arg must be file");
1085                 return NULL;
1086         }
1087         rf.fp = PyFile_AsFile(f);
1088         rf.strings = PyList_New(0);
1089         result = read_object(&rf);
1090         Py_DECREF(rf.strings);
1091         return result;
1092 }
1093
1094 static PyObject *
1095 marshal_dumps(PyObject *self, PyObject *args)
1096 {
1097         PyObject *x;
1098         int version = Py_MARSHAL_VERSION;
1099         if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1100                 return NULL;
1101         return PyMarshal_WriteObjectToString(x, version);
1102 }
1103
1104 static PyObject *
1105 marshal_loads(PyObject *self, PyObject *args)
1106 {
1107         RFILE rf;
1108         char *s;
1109         Py_ssize_t n;
1110         PyObject* result;
1111         if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1112                 return NULL;
1113         rf.fp = NULL;
1114         rf.ptr = s;
1115         rf.end = s + n;
1116         rf.strings = PyList_New(0);
1117         result = read_object(&rf);
1118         Py_DECREF(rf.strings);
1119         return result;
1120 }
1121
1122 static PyMethodDef marshal_methods[] = {
1123         {"dump",        marshal_dump,   METH_VARARGS},
1124         {"load",        marshal_load,   METH_VARARGS},
1125         {"dumps",       marshal_dumps,  METH_VARARGS},
1126         {"loads",       marshal_loads,  METH_VARARGS},
1127         {NULL,          NULL}           /* sentinel */
1128 };
1129
1130 PyMODINIT_FUNC
1131 PyMarshal_Init(void)
1132 {
1133         PyObject *mod = Py_InitModule("marshal", marshal_methods);
1134         if (mod == NULL)
1135                 return;
1136         PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1137 }