?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Fix issue number in comment.
[python.git] / Python / marshal.c
blobc7015d2eb7b5960a326bd2e82367770b1ef060ae
1
2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
6
7 #define PY_SSIZE_T_CLEAN
8
9 #include "Python.h"
10 #include "longintrepr.h"
11 #include "code.h"
12 #include "marshal.h"
13
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
15
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
19 */
20 #define MAX_MARSHAL_STACK_DEPTH 2000
21
22 #define TYPE_NULL '0'
23 #define TYPE_NONE 'N'
24 #define TYPE_FALSE 'F'
25 #define TYPE_TRUE 'T'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
28 #define TYPE_INT 'i'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
34 #define TYPE_LONG 'l'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
39 #define TYPE_LIST '['
40 #define TYPE_DICT '{'
41 #define TYPE_CODE 'c'
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
44 #define TYPE_SET '<'
45 #define TYPE_FROZENSET '>'
46
47 #define WFERR_OK 0
48 #define WFERR_UNMARSHALLABLE 1
49 #define WFERR_NESTEDTOODEEP 2
50 #define WFERR_NOMEMORY 3
51
52 typedef struct {
53 FILE *fp;
54 int error; /* see WFERR_* values */
55 int depth;
56 /* If fp == NULL, the following are valid: */
57 PyObject *str;
58 char *ptr;
59 char *end;
60 PyObject *strings; /* dict on marshal, list on unmarshal */
61 int version;
62 } WFILE;
63
64 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
65 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
66 else w_more(c, p)
67
68 static void
69 w_more(int c, WFILE *p)
70 {
71 Py_ssize_t size, newsize;
72 if (p->str == NULL)
73 return; /* An error already occurred */
74 size = PyString_Size(p->str);
75 newsize = size + size + 1024;
76 if (newsize > 32*1024*1024) {
77 newsize = size + (size >> 3); /* 12.5% overallocation */
78 }
79 if (_PyString_Resize(&p->str, newsize) != 0) {
80 p->ptr = p->end = NULL;
81 }
82 else {
83 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
84 p->end =
85 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
86 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
87 }
88 }
89
90 static void
91 w_string(char *s, int n, WFILE *p)
92 {
93 if (p->fp != NULL) {
94 fwrite(s, 1, n, p->fp);
95 }
96 else {
97 while (--n >= 0) {
98 w_byte(*s, p);
99 s++;
100 }
101 }
102 }
103
104 static void
105 w_short(int x, WFILE *p)
106 {
107 w_byte((char)( x & 0xff), p);
108 w_byte((char)((x>> 8) & 0xff), p);
109 }
110
111 static void
112 w_long(long x, WFILE *p)
113 {
114 w_byte((char)( x & 0xff), p);
115 w_byte((char)((x>> 8) & 0xff), p);
116 w_byte((char)((x>>16) & 0xff), p);
117 w_byte((char)((x>>24) & 0xff), p);
118 }
119
120 #if SIZEOF_LONG > 4
121 static void
122 w_long64(long x, WFILE *p)
123 {
124 w_long(x, p);
125 w_long(x>>32, p);
126 }
127 #endif
128
129 /* We assume that Python longs are stored internally in base some power of
130 2**15; for the sake of portability we'll always read and write them in base
131 exactly 2**15. */
132
133 #define PyLong_MARSHAL_SHIFT 15
134 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
135 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
136 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
137 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
138 #endif
139 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
140
141 static void
142 w_PyLong(const PyLongObject *ob, WFILE *p)
143 {
144 Py_ssize_t i, j, n, l;
145 digit d;
146
147 w_byte(TYPE_LONG, p);
148 if (Py_SIZE(ob) == 0) {
149 w_long((long)0, p);
150 return;
151 }
152
153 /* set l to number of base PyLong_MARSHAL_BASE digits */
154 n = ABS(Py_SIZE(ob));
155 l = (n-1) * PyLong_MARSHAL_RATIO;
156 d = ob->ob_digit[n-1];
157 assert(d != 0); /* a PyLong is always normalized */
158 do {
159 d >>= PyLong_MARSHAL_SHIFT;
160 l++;
161 } while (d != 0);
162 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
163
164 for (i=0; i < n-1; i++) {
165 d = ob->ob_digit[i];
166 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
167 w_short(d & PyLong_MARSHAL_MASK, p);
168 d >>= PyLong_MARSHAL_SHIFT;
169 }
170 assert (d == 0);
171 }
172 d = ob->ob_digit[n-1];
173 do {
174 w_short(d & PyLong_MARSHAL_MASK, p);
175 d >>= PyLong_MARSHAL_SHIFT;
176 } while (d != 0);
177 }
178
179 static void
180 w_object(PyObject *v, WFILE *p)
181 {
182 Py_ssize_t i, n;
183
184 p->depth++;
185
186 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
187 p->error = WFERR_NESTEDTOODEEP;
188 }
189 else if (v == NULL) {
190 w_byte(TYPE_NULL, p);
191 }
192 else if (v == Py_None) {
193 w_byte(TYPE_NONE, p);
194 }
195 else if (v == PyExc_StopIteration) {
196 w_byte(TYPE_STOPITER, p);
197 }
198 else if (v == Py_Ellipsis) {
199 w_byte(TYPE_ELLIPSIS, p);
200 }
201 else if (v == Py_False) {
202 w_byte(TYPE_FALSE, p);
203 }
204 else if (v == Py_True) {
205 w_byte(TYPE_TRUE, p);
206 }
207 else if (PyInt_CheckExact(v)) {
208 long x = PyInt_AS_LONG((PyIntObject *)v);
209 #if SIZEOF_LONG > 4
210 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
211 if (y && y != -1) {
212 w_byte(TYPE_INT64, p);
213 w_long64(x, p);
214 }
215 else
216 #endif
217 {
218 w_byte(TYPE_INT, p);
219 w_long(x, p);
220 }
221 }
222 else if (PyLong_CheckExact(v)) {
223 PyLongObject *ob = (PyLongObject *)v;
224 w_PyLong(ob, p);
225 }
226 else if (PyFloat_CheckExact(v)) {
227 if (p->version > 1) {
228 unsigned char buf[8];
229 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
230 buf, 1) < 0) {
231 p->error = WFERR_UNMARSHALLABLE;
232 return;
233 }
234 w_byte(TYPE_BINARY_FLOAT, p);
235 w_string((char*)buf, 8, p);
236 }
237 else {
238 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
239 'g', 17, 0, NULL);
240 if (!buf) {
241 p->error = WFERR_NOMEMORY;
242 return;
243 }
244 n = strlen(buf);
245 w_byte(TYPE_FLOAT, p);
246 w_byte((int)n, p);
247 w_string(buf, (int)n, p);
248 PyMem_Free(buf);
249 }
250 }
251 #ifndef WITHOUT_COMPLEX
252 else if (PyComplex_CheckExact(v)) {
253 if (p->version > 1) {
254 unsigned char buf[8];
255 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
256 buf, 1) < 0) {
257 p->error = WFERR_UNMARSHALLABLE;
258 return;
259 }
260 w_byte(TYPE_BINARY_COMPLEX, p);
261 w_string((char*)buf, 8, p);
262 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
263 buf, 1) < 0) {
264 p->error = WFERR_UNMARSHALLABLE;
265 return;
266 }
267 w_string((char*)buf, 8, p);
268 }
269 else {
270 char *buf;
271 w_byte(TYPE_COMPLEX, p);
272 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
273 'g', 17, 0, NULL);
274 if (!buf) {
275 p->error = WFERR_NOMEMORY;
276 return;
277 }
278 n = strlen(buf);
279 w_byte((int)n, p);
280 w_string(buf, (int)n, p);
281 PyMem_Free(buf);
282 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
283 'g', 17, 0, NULL);
284 if (!buf) {
285 p->error = WFERR_NOMEMORY;
286 return;
287 }
288 n = strlen(buf);
289 w_byte((int)n, p);
290 w_string(buf, (int)n, p);
291 PyMem_Free(buf);
292 }
293 }
294 #endif
295 else if (PyString_CheckExact(v)) {
296 if (p->strings && PyString_CHECK_INTERNED(v)) {
297 PyObject *o = PyDict_GetItem(p->strings, v);
298 if (o) {
299 long w = PyInt_AsLong(o);
300 w_byte(TYPE_STRINGREF, p);
301 w_long(w, p);
302 goto exit;
303 }
304 else {
305 int ok;
306 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
307 ok = o &&
308 PyDict_SetItem(p->strings, v, o) >= 0;
309 Py_XDECREF(o);
310 if (!ok) {
311 p->depth--;
312 p->error = WFERR_UNMARSHALLABLE;
313 return;
314 }
315 w_byte(TYPE_INTERNED, p);
316 }
317 }
318 else {
319 w_byte(TYPE_STRING, p);
320 }
321 n = PyString_GET_SIZE(v);
322 if (n > INT_MAX) {
323 /* huge strings are not supported */
324 p->depth--;
325 p->error = WFERR_UNMARSHALLABLE;
326 return;
327 }
328 w_long((long)n, p);
329 w_string(PyString_AS_STRING(v), (int)n, p);
330 }
331 #ifdef Py_USING_UNICODE
332 else if (PyUnicode_CheckExact(v)) {
333 PyObject *utf8;
334 utf8 = PyUnicode_AsUTF8String(v);
335 if (utf8 == NULL) {
336 p->depth--;
337 p->error = WFERR_UNMARSHALLABLE;
338 return;
339 }
340 w_byte(TYPE_UNICODE, p);
341 n = PyString_GET_SIZE(utf8);
342 if (n > INT_MAX) {
343 p->depth--;
344 p->error = WFERR_UNMARSHALLABLE;
345 return;
346 }
347 w_long((long)n, p);
348 w_string(PyString_AS_STRING(utf8), (int)n, p);
349 Py_DECREF(utf8);
350 }
351 #endif
352 else if (PyTuple_CheckExact(v)) {
353 w_byte(TYPE_TUPLE, p);
354 n = PyTuple_Size(v);
355 w_long((long)n, p);
356 for (i = 0; i < n; i++) {
357 w_object(PyTuple_GET_ITEM(v, i), p);
358 }
359 }
360 else if (PyList_CheckExact(v)) {
361 w_byte(TYPE_LIST, p);
362 n = PyList_GET_SIZE(v);
363 w_long((long)n, p);
364 for (i = 0; i < n; i++) {
365 w_object(PyList_GET_ITEM(v, i), p);
366 }
367 }
368 else if (PyDict_CheckExact(v)) {
369 Py_ssize_t pos;
370 PyObject *key, *value;
371 w_byte(TYPE_DICT, p);
372 /* This one is NULL object terminated! */
373 pos = 0;
374 while (PyDict_Next(v, &pos, &key, &value)) {
375 w_object(key, p);
376 w_object(value, p);
377 }
378 w_object((PyObject *)NULL, p);
379 }
380 else if (PyAnySet_CheckExact(v)) {
381 PyObject *value, *it;
382
383 if (PyObject_TypeCheck(v, &PySet_Type))
384 w_byte(TYPE_SET, p);
385 else
386 w_byte(TYPE_FROZENSET, p);
387 n = PyObject_Size(v);
388 if (n == -1) {
389 p->depth--;
390 p->error = WFERR_UNMARSHALLABLE;
391 return;
392 }
393 w_long((long)n, p);
394 it = PyObject_GetIter(v);
395 if (it == NULL) {
396 p->depth--;
397 p->error = WFERR_UNMARSHALLABLE;
398 return;
399 }
400 while ((value = PyIter_Next(it)) != NULL) {
401 w_object(value, p);
402 Py_DECREF(value);
403 }
404 Py_DECREF(it);
405 if (PyErr_Occurred()) {
406 p->depth--;
407 p->error = WFERR_UNMARSHALLABLE;
408 return;
409 }
410 }
411 else if (PyCode_Check(v)) {
412 PyCodeObject *co = (PyCodeObject *)v;
413 w_byte(TYPE_CODE, p);
414 w_long(co->co_argcount, p);
415 w_long(co->co_nlocals, p);
416 w_long(co->co_stacksize, p);
417 w_long(co->co_flags, p);
418 w_object(co->co_code, p);
419 w_object(co->co_consts, p);
420 w_object(co->co_names, p);
421 w_object(co->co_varnames, p);
422 w_object(co->co_freevars, p);
423 w_object(co->co_cellvars, p);
424 w_object(co->co_filename, p);
425 w_object(co->co_name, p);
426 w_long(co->co_firstlineno, p);
427 w_object(co->co_lnotab, p);
428 }
429 else if (PyObject_CheckReadBuffer(v)) {
430 /* Write unknown buffer-style objects as a string */
431 char *s;
432 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
433 w_byte(TYPE_STRING, p);
434 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
435 if (n > INT_MAX) {
436 p->depth--;
437 p->error = WFERR_UNMARSHALLABLE;
438 return;
439 }
440 w_long((long)n, p);
441 w_string(s, (int)n, p);
442 }
443 else {
444 w_byte(TYPE_UNKNOWN, p);
445 p->error = WFERR_UNMARSHALLABLE;
446 }
447 exit:
448 p->depth--;
449 }
450
451 /* version currently has no effect for writing longs. */
452 void
453 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
454 {
455 WFILE wf;
456 wf.fp = fp;
457 wf.error = WFERR_OK;
458 wf.depth = 0;
459 wf.strings = NULL;
460 wf.version = version;
461 w_long(x, &wf);
462 }
463
464 void
465 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
466 {
467 WFILE wf;
468 wf.fp = fp;
469 wf.error = WFERR_OK;
470 wf.depth = 0;
471 wf.strings = (version > 0) ? PyDict_New() : NULL;
472 wf.version = version;
473 w_object(x, &wf);
474 Py_XDECREF(wf.strings);
475 }
476
477 typedef WFILE RFILE; /* Same struct with different invariants */
478
479 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
480
481 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
482
483 static int
484 r_string(char *s, int n, RFILE *p)
485 {
486 if (p->fp != NULL)
487 /* The result fits into int because it must be <=n. */
488 return (int)fread(s, 1, n, p->fp);
489 if (p->end - p->ptr < n)
490 n = (int)(p->end - p->ptr);
491 memcpy(s, p->ptr, n);
492 p->ptr += n;
493 return n;
494 }
495
496 static int
497 r_short(RFILE *p)
498 {
499 register short x;
500 x = r_byte(p);
501 x |= r_byte(p) << 8;
502 /* Sign-extension, in case short greater than 16 bits */
503 x |= -(x & 0x8000);
504 return x;
505 }
506
507 static long
508 r_long(RFILE *p)
509 {
510 register long x;
511 register FILE *fp = p->fp;
512 if (fp) {
513 x = getc(fp);
514 x |= (long)getc(fp) << 8;
515 x |= (long)getc(fp) << 16;
516 x |= (long)getc(fp) << 24;
517 }
518 else {
519 x = rs_byte(p);
520 x |= (long)rs_byte(p) << 8;
521 x |= (long)rs_byte(p) << 16;
522 x |= (long)rs_byte(p) << 24;
523 }
524 #if SIZEOF_LONG > 4
525 /* Sign extension for 64-bit machines */
526 x |= -(x & 0x80000000L);
527 #endif
528 return x;
529 }
530
531 /* r_long64 deals with the TYPE_INT64 code. On a machine with
532 sizeof(long) > 4, it returns a Python int object, else a Python long
533 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
534 so there's no inefficiency here in returning a PyLong on 32-bit boxes
535 for everything written via TYPE_INT64 (i.e., if an int is written via
536 TYPE_INT64, it *needs* more than 32 bits).
537 */
538 static PyObject *
539 r_long64(RFILE *p)
540 {
541 long lo4 = r_long(p);
542 long hi4 = r_long(p);
543 #if SIZEOF_LONG > 4
544 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
545 return PyInt_FromLong(x);
546 #else
547 unsigned char buf[8];
548 int one = 1;
549 int is_little_endian = (int)*(char*)&one;
550 if (is_little_endian) {
551 memcpy(buf, &lo4, 4);
552 memcpy(buf+4, &hi4, 4);
553 }
554 else {
555 memcpy(buf, &hi4, 4);
556 memcpy(buf+4, &lo4, 4);
557 }
558 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
559 #endif
560 }
561
562 static PyObject *
563 r_PyLong(RFILE *p)
564 {
565 PyLongObject *ob;
566 int size, i, j, md, shorts_in_top_digit;
567 long n;
568 digit d;
569
570 n = r_long(p);
571 if (n == 0)
572 return (PyObject *)_PyLong_New(0);
573 if (n < -INT_MAX || n > INT_MAX) {
574 PyErr_SetString(PyExc_ValueError,
575 "bad marshal data (long size out of range)");
576 return NULL;
577 }
578
579 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
580 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
581 ob = _PyLong_New(size);
582 if (ob == NULL)
583 return NULL;
584 Py_SIZE(ob) = n > 0 ? size : -size;
585
586 for (i = 0; i < size-1; i++) {
587 d = 0;
588 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
589 md = r_short(p);
590 if (md < 0 || md > PyLong_MARSHAL_BASE)
591 goto bad_digit;
592 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
593 }
594 ob->ob_digit[i] = d;
595 }
596 d = 0;
597 for (j=0; j < shorts_in_top_digit; j++) {
598 md = r_short(p);
599 if (md < 0 || md > PyLong_MARSHAL_BASE)
600 goto bad_digit;
601 /* topmost marshal digit should be nonzero */
602 if (md == 0 && j == shorts_in_top_digit - 1) {
603 Py_DECREF(ob);
604 PyErr_SetString(PyExc_ValueError,
605 "bad marshal data (unnormalized long data)");
606 return NULL;
607 }
608 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
609 }
610 /* top digit should be nonzero, else the resulting PyLong won't be
611 normalized */
612 ob->ob_digit[size-1] = d;
613 return (PyObject *)ob;
614 bad_digit:
615 Py_DECREF(ob);
616 PyErr_SetString(PyExc_ValueError,
617 "bad marshal data (digit out of range in long)");
618 return NULL;
619 }
620
621
622 static PyObject *
623 r_object(RFILE *p)
624 {
625 /* NULL is a valid return value, it does not necessarily means that
626 an exception is set. */
627 PyObject *v, *v2;
628 long i, n;
629 int type = r_byte(p);
630 PyObject *retval;
631
632 p->depth++;
633
634 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
635 p->depth--;
636 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
637 return NULL;
638 }
639
640 switch (type) {
641
642 case EOF:
643 PyErr_SetString(PyExc_EOFError,
644 "EOF read where object expected");
645 retval = NULL;
646 break;
647
648 case TYPE_NULL:
649 retval = NULL;
650 break;
651
652 case TYPE_NONE:
653 Py_INCREF(Py_None);
654 retval = Py_None;
655 break;
656
657 case TYPE_STOPITER:
658 Py_INCREF(PyExc_StopIteration);
659 retval = PyExc_StopIteration;
660 break;
661
662 case TYPE_ELLIPSIS:
663 Py_INCREF(Py_Ellipsis);
664 retval = Py_Ellipsis;
665 break;
666
667 case TYPE_FALSE:
668 Py_INCREF(Py_False);
669 retval = Py_False;
670 break;
671
672 case TYPE_TRUE:
673 Py_INCREF(Py_True);
674 retval = Py_True;
675 break;
676
677 case TYPE_INT:
678 retval = PyInt_FromLong(r_long(p));
679 break;
680
681 case TYPE_INT64:
682 retval = r_long64(p);
683 break;
684
685 case TYPE_LONG:
686 retval = r_PyLong(p);
687 break;
688
689 case TYPE_FLOAT:
690 {
691 char buf[256];
692 double dx;
693 n = r_byte(p);
694 if (n == EOF || r_string(buf, (int)n, p) != n) {
695 PyErr_SetString(PyExc_EOFError,
696 "EOF read where object expected");
697 retval = NULL;
698 break;
699 }
700 buf[n] = '\0';
701 dx = PyOS_string_to_double(buf, NULL, NULL);
702 if (dx == -1.0 && PyErr_Occurred()) {
703 retval = NULL;
704 break;
705 }
706 retval = PyFloat_FromDouble(dx);
707 break;
708 }
709
710 case TYPE_BINARY_FLOAT:
711 {
712 unsigned char buf[8];
713 double x;
714 if (r_string((char*)buf, 8, p) != 8) {
715 PyErr_SetString(PyExc_EOFError,
716 "EOF read where object expected");
717 retval = NULL;
718 break;
719 }
720 x = _PyFloat_Unpack8(buf, 1);
721 if (x == -1.0 && PyErr_Occurred()) {
722 retval = NULL;
723 break;
724 }
725 retval = PyFloat_FromDouble(x);
726 break;
727 }
728
729 #ifndef WITHOUT_COMPLEX
730 case TYPE_COMPLEX:
731 {
732 char buf[256];
733 Py_complex c;
734 n = r_byte(p);
735 if (n == EOF || r_string(buf, (int)n, p) != n) {
736 PyErr_SetString(PyExc_EOFError,
737 "EOF read where object expected");
738 retval = NULL;
739 break;
740 }
741 buf[n] = '\0';
742 c.real = PyOS_string_to_double(buf, NULL, NULL);
743 if (c.real == -1.0 && PyErr_Occurred()) {
744 retval = NULL;
745 break;
746 }
747 n = r_byte(p);
748 if (n == EOF || r_string(buf, (int)n, p) != n) {
749 PyErr_SetString(PyExc_EOFError,
750 "EOF read where object expected");
751 retval = NULL;
752 break;
753 }
754 buf[n] = '\0';
755 c.imag = PyOS_string_to_double(buf, NULL, NULL);
756 if (c.imag == -1.0 && PyErr_Occurred()) {
757 retval = NULL;
758 break;
759 }
760 retval = PyComplex_FromCComplex(c);
761 break;
762 }
763
764 case TYPE_BINARY_COMPLEX:
765 {
766 unsigned char buf[8];
767 Py_complex c;
768 if (r_string((char*)buf, 8, p) != 8) {
769 PyErr_SetString(PyExc_EOFError,
770 "EOF read where object expected");
771 retval = NULL;
772 break;
773 }
774 c.real = _PyFloat_Unpack8(buf, 1);
775 if (c.real == -1.0 && PyErr_Occurred()) {
776 retval = NULL;
777 break;
778 }
779 if (r_string((char*)buf, 8, p) != 8) {
780 PyErr_SetString(PyExc_EOFError,
781 "EOF read where object expected");
782 retval = NULL;
783 break;
784 }
785 c.imag = _PyFloat_Unpack8(buf, 1);
786 if (c.imag == -1.0 && PyErr_Occurred()) {
787 retval = NULL;
788 break;
789 }
790 retval = PyComplex_FromCComplex(c);
791 break;
792 }
793 #endif
794
795 case TYPE_INTERNED:
796 case TYPE_STRING:
797 n = r_long(p);
798 if (n < 0 || n > INT_MAX) {
799 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
800 retval = NULL;
801 break;
802 }
803 v = PyString_FromStringAndSize((char *)NULL, n);
804 if (v == NULL) {
805 retval = NULL;
806 break;
807 }
808 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
809 Py_DECREF(v);
810 PyErr_SetString(PyExc_EOFError,
811 "EOF read where object expected");
812 retval = NULL;
813 break;
814 }
815 if (type == TYPE_INTERNED) {
816 PyString_InternInPlace(&v);
817 if (PyList_Append(p->strings, v) < 0) {
818 retval = NULL;
819 break;
820 }
821 }
822 retval = v;
823 break;
824
825 case TYPE_STRINGREF:
826 n = r_long(p);
827 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
828 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)");
829 retval = NULL;
830 break;
831 }
832 v = PyList_GET_ITEM(p->strings, n);
833 Py_INCREF(v);
834 retval = v;
835 break;
836
837 #ifdef Py_USING_UNICODE
838 case TYPE_UNICODE:
839 {
840 char *buffer;
841
842 n = r_long(p);
843 if (n < 0 || n > INT_MAX) {
844 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
845 retval = NULL;
846 break;
847 }
848 buffer = PyMem_NEW(char, n);
849 if (buffer == NULL) {
850 retval = PyErr_NoMemory();
851 break;
852 }
853 if (r_string(buffer, (int)n, p) != n) {
854 PyMem_DEL(buffer);
855 PyErr_SetString(PyExc_EOFError,
856 "EOF read where object expected");
857 retval = NULL;
858 break;
859 }
860 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
861 PyMem_DEL(buffer);
862 retval = v;
863 break;
864 }
865 #endif
866
867 case TYPE_TUPLE:
868 n = r_long(p);
869 if (n < 0 || n > INT_MAX) {
870 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
871 retval = NULL;
872 break;
873 }
874 v = PyTuple_New((int)n);
875 if (v == NULL) {
876 retval = NULL;
877 break;
878 }
879 for (i = 0; i < n; i++) {
880 v2 = r_object(p);
881 if ( v2 == NULL ) {
882 if (!PyErr_Occurred())
883 PyErr_SetString(PyExc_TypeError,
884 "NULL object in marshal data for tuple");
885 Py_DECREF(v);
886 v = NULL;
887 break;
888 }
889 PyTuple_SET_ITEM(v, (int)i, v2);
890 }
891 retval = v;
892 break;
893
894 case TYPE_LIST:
895 n = r_long(p);
896 if (n < 0 || n > INT_MAX) {
897 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
898 retval = NULL;
899 break;
900 }
901 v = PyList_New((int)n);
902 if (v == NULL) {
903 retval = NULL;
904 break;
905 }
906 for (i = 0; i < n; i++) {
907 v2 = r_object(p);
908 if ( v2 == NULL ) {
909 if (!PyErr_Occurred())
910 PyErr_SetString(PyExc_TypeError,
911 "NULL object in marshal data for list");
912 Py_DECREF(v);
913 v = NULL;
914 break;
915 }
916 PyList_SET_ITEM(v, (int)i, v2);
917 }
918 retval = v;
919 break;
920
921 case TYPE_DICT:
922 v = PyDict_New();
923 if (v == NULL) {
924 retval = NULL;
925 break;
926 }
927 for (;;) {
928 PyObject *key, *val;
929 key = r_object(p);
930 if (key == NULL)
931 break;
932 val = r_object(p);
933 if (val != NULL)
934 PyDict_SetItem(v, key, val);
935 Py_DECREF(key);
936 Py_XDECREF(val);
937 }
938 if (PyErr_Occurred()) {
939 Py_DECREF(v);
940 v = NULL;
941 }
942 retval = v;
943 break;
944
945 case TYPE_SET:
946 case TYPE_FROZENSET:
947 n = r_long(p);
948 if (n < 0 || n > INT_MAX) {
949 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
950 retval = NULL;
951 break;
952 }
953 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
954 if (v == NULL) {
955 retval = NULL;
956 break;
957 }
958 for (i = 0; i < n; i++) {
959 v2 = r_object(p);
960 if ( v2 == NULL ) {
961 if (!PyErr_Occurred())
962 PyErr_SetString(PyExc_TypeError,
963 "NULL object in marshal data for set");
964 Py_DECREF(v);
965 v = NULL;
966 break;
967 }
968 if (PySet_Add(v, v2) == -1) {
969 Py_DECREF(v);
970 Py_DECREF(v2);
971 v = NULL;
972 break;
973 }
974 Py_DECREF(v2);
975 }
976 retval = v;
977 break;
978
979 case TYPE_CODE:
980 if (PyEval_GetRestricted()) {
981 PyErr_SetString(PyExc_RuntimeError,
982 "cannot unmarshal code objects in "
983 "restricted execution mode");
984 retval = NULL;
985 break;
986 }
987 else {
988 int argcount;
989 int nlocals;
990 int stacksize;
991 int flags;
992 PyObject *code = NULL;
993 PyObject *consts = NULL;
994 PyObject *names = NULL;
995 PyObject *varnames = NULL;
996 PyObject *freevars = NULL;
997 PyObject *cellvars = NULL;
998 PyObject *filename = NULL;
999 PyObject *name = NULL;
1000 int firstlineno;
1001 PyObject *lnotab = NULL;
1002
1003 v = NULL;
1004
1005 /* XXX ignore long->int overflows for now */
1006 argcount = (int)r_long(p);
1007 nlocals = (int)r_long(p);
1008 stacksize = (int)r_long(p);
1009 flags = (int)r_long(p);
1010 code = r_object(p);
1011 if (code == NULL)
1012 goto code_error;
1013 consts = r_object(p);
1014 if (consts == NULL)
1015 goto code_error;
1016 names = r_object(p);
1017 if (names == NULL)
1018 goto code_error;
1019 varnames = r_object(p);
1020 if (varnames == NULL)
1021 goto code_error;
1022 freevars = r_object(p);
1023 if (freevars == NULL)
1024 goto code_error;
1025 cellvars = r_object(p);
1026 if (cellvars == NULL)
1027 goto code_error;
1028 filename = r_object(p);
1029 if (filename == NULL)
1030 goto code_error;
1031 name = r_object(p);
1032 if (name == NULL)
1033 goto code_error;
1034 firstlineno = (int)r_long(p);
1035 lnotab = r_object(p);
1036 if (lnotab == NULL)
1037 goto code_error;
1038
1039 v = (PyObject *) PyCode_New(
1040 argcount, nlocals, stacksize, flags,
1041 code, consts, names, varnames,
1042 freevars, cellvars, filename, name,
1043 firstlineno, lnotab);
1044
1045 code_error:
1046 Py_XDECREF(code);
1047 Py_XDECREF(consts);
1048 Py_XDECREF(names);
1049 Py_XDECREF(varnames);
1050 Py_XDECREF(freevars);
1051 Py_XDECREF(cellvars);
1052 Py_XDECREF(filename);
1053 Py_XDECREF(name);
1054 Py_XDECREF(lnotab);
1055
1056 }
1057 retval = v;
1058 break;
1059
1060 default:
1061 /* Bogus data got written, which isn't ideal.
1062 This will let you keep working and recover. */
1063 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1064 retval = NULL;
1065 break;
1066
1067 }
1068 p->depth--;
1069 return retval;
1070 }
1071
1072 static PyObject *
1073 read_object(RFILE *p)
1074 {
1075 PyObject *v;
1076 if (PyErr_Occurred()) {
1077 fprintf(stderr, "XXX readobject called with exception set\n");
1078 return NULL;
1079 }
1080 v = r_object(p);
1081 if (v == NULL && !PyErr_Occurred())
1082 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1083 return v;
1084 }
1085
1086 int
1087 PyMarshal_ReadShortFromFile(FILE *fp)
1088 {
1089 RFILE rf;
1090 assert(fp);
1091 rf.fp = fp;
1092 rf.strings = NULL;
1093 rf.end = rf.ptr = NULL;
1094 return r_short(&rf);
1095 }
1096
1097 long
1098 PyMarshal_ReadLongFromFile(FILE *fp)
1099 {
1100 RFILE rf;
1101 rf.fp = fp;
1102 rf.strings = NULL;
1103 rf.ptr = rf.end = NULL;
1104 return r_long(&rf);
1105 }
1106
1107 #ifdef HAVE_FSTAT
1108 /* Return size of file in bytes; < 0 if unknown. */
1109 static off_t
1110 getfilesize(FILE *fp)
1111 {
1112 struct stat st;
1113 if (fstat(fileno(fp), &st) != 0)
1114 return -1;
1115 else
1116 return st.st_size;
1117 }
1118 #endif
1119
1120 /* If we can get the size of the file up-front, and it's reasonably small,
1121 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1122 * than reading a byte at a time from file; speeds .pyc imports.
1123 * CAUTION: since this may read the entire remainder of the file, don't
1124 * call it unless you know you're done with the file.
1125 */
1126 PyObject *
1127 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1128 {
1129 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1130 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1131 */
1132 #define SMALL_FILE_LIMIT (1L << 14)
1133 #define REASONABLE_FILE_LIMIT (1L << 18)
1134 #ifdef HAVE_FSTAT
1135 off_t filesize;
1136 #endif
1137 #ifdef HAVE_FSTAT
1138 filesize = getfilesize(fp);
1139 if (filesize > 0) {
1140 char buf[SMALL_FILE_LIMIT];
1141 char* pBuf = NULL;
1142 if (filesize <= SMALL_FILE_LIMIT)
1143 pBuf = buf;
1144 else if (filesize <= REASONABLE_FILE_LIMIT)
1145 pBuf = (char *)PyMem_MALLOC(filesize);
1146 if (pBuf != NULL) {
1147 PyObject* v;
1148 size_t n;
1149 /* filesize must fit into an int, because it
1150 is smaller than REASONABLE_FILE_LIMIT */
1151 n = fread(pBuf, 1, (int)filesize, fp);
1152 v = PyMarshal_ReadObjectFromString(pBuf, n);
1153 if (pBuf != buf)
1154 PyMem_FREE(pBuf);
1155 return v;
1156 }
1157
1158 }
1159 #endif
1160 /* We don't have fstat, or we do but the file is larger than
1161 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1162 */
1163 return PyMarshal_ReadObjectFromFile(fp);
1164
1165 #undef SMALL_FILE_LIMIT
1166 #undef REASONABLE_FILE_LIMIT
1167 }
1168
1169 PyObject *
1170 PyMarshal_ReadObjectFromFile(FILE *fp)
1171 {
1172 RFILE rf;
1173 PyObject *result;
1174 rf.fp = fp;
1175 rf.strings = PyList_New(0);
1176 rf.depth = 0;
1177 rf.ptr = rf.end = NULL;
1178 result = r_object(&rf);
1179 Py_DECREF(rf.strings);
1180 return result;
1181 }
1182
1183 PyObject *
1184 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1185 {
1186 RFILE rf;
1187 PyObject *result;
1188 rf.fp = NULL;
1189 rf.ptr = str;
1190 rf.end = str + len;
1191 rf.strings = PyList_New(0);
1192 rf.depth = 0;
1193 result = r_object(&rf);
1194 Py_DECREF(rf.strings);
1195 return result;
1196 }
1197
1198 static void
1199 set_error(int error)
1200 {
1201 switch (error) {
1202 case WFERR_NOMEMORY:
1203 PyErr_NoMemory();
1204 break;
1205 case WFERR_UNMARSHALLABLE:
1206 PyErr_SetString(PyExc_ValueError, "unmarshallable object");
1207 break;
1208 case WFERR_NESTEDTOODEEP:
1209 default:
1210 PyErr_SetString(PyExc_ValueError,
1211 "object too deeply nested to marshal");
1212 break;
1213 }
1214 }
1215
1216 PyObject *
1217 PyMarshal_WriteObjectToString(PyObject *x, int version)
1218 {
1219 WFILE wf;
1220 wf.fp = NULL;
1221 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1222 if (wf.str == NULL)
1223 return NULL;
1224 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1225 wf.end = wf.ptr + PyString_Size(wf.str);
1226 wf.error = WFERR_OK;
1227 wf.depth = 0;
1228 wf.version = version;
1229 wf.strings = (version > 0) ? PyDict_New() : NULL;
1230 w_object(x, &wf);
1231 Py_XDECREF(wf.strings);
1232 if (wf.str != NULL) {
1233 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1234 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1235 Py_DECREF(wf.str);
1236 PyErr_SetString(PyExc_OverflowError,
1237 "too much marshall data for a string");
1238 return NULL;
1239 }
1240 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1241 }
1242 if (wf.error != WFERR_OK) {
1243 Py_XDECREF(wf.str);
1244 set_error(wf.error);
1245 return NULL;
1246 }
1247 return wf.str;
1248 }
1249
1250 /* And an interface for Python programs... */
1251
1252 static PyObject *
1253 marshal_dump(PyObject *self, PyObject *args)
1254 {
1255 WFILE wf;
1256 PyObject *x;
1257 PyObject *f;
1258 int version = Py_MARSHAL_VERSION;
1259 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1260 return NULL;
1261 if (!PyFile_Check(f)) {
1262 PyErr_SetString(PyExc_TypeError,
1263 "marshal.dump() 2nd arg must be file");
1264 return NULL;
1265 }
1266 wf.fp = PyFile_AsFile(f);
1267 wf.str = NULL;
1268 wf.ptr = wf.end = NULL;
1269 wf.error = WFERR_OK;
1270 wf.depth = 0;
1271 wf.strings = (version > 0) ? PyDict_New() : 0;
1272 wf.version = version;
1273 w_object(x, &wf);
1274 Py_XDECREF(wf.strings);
1275 if (wf.error != WFERR_OK) {
1276 set_error(wf.error);
1277 return NULL;
1278 }
1279 Py_INCREF(Py_None);
1280 return Py_None;
1281 }
1282
1283 PyDoc_STRVAR(dump_doc,
1284 "dump(value, file[, version])\n\
1285 \n\
1286 Write the value on the open file. The value must be a supported type.\n\
1287 The file must be an open file object such as sys.stdout or returned by\n\
1288 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1289 \n\
1290 If the value has (or contains an object that has) an unsupported type, a\n\
1291 ValueError exception is raised — but garbage data will also be written\n\
1292 to the file. The object will not be properly read back by load()\n\
1293 \n\
1294 New in version 2.4: The version argument indicates the data format that\n\
1295 dump should use.");
1296
1297 static PyObject *
1298 marshal_load(PyObject *self, PyObject *f)
1299 {
1300 RFILE rf;
1301 PyObject *result;
1302 if (!PyFile_Check(f)) {
1303 PyErr_SetString(PyExc_TypeError,
1304 "marshal.load() arg must be file");
1305 return NULL;
1306 }
1307 rf.fp = PyFile_AsFile(f);
1308 rf.strings = PyList_New(0);
1309 rf.depth = 0;
1310 result = read_object(&rf);
1311 Py_DECREF(rf.strings);
1312 return result;
1313 }
1314
1315 PyDoc_STRVAR(load_doc,
1316 "load(file)\n\
1317 \n\
1318 Read one value from the open file and return it. If no valid value is\n\
1319 read (e.g. because the data has a different Python version’s\n\
1320 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1321 The file must be an open file object opened in binary mode ('rb' or\n\
1322 'r+b').\n\
1323 \n\
1324 Note: If an object containing an unsupported type was marshalled with\n\
1325 dump(), load() will substitute None for the unmarshallable type.");
1326
1327
1328 static PyObject *
1329 marshal_dumps(PyObject *self, PyObject *args)
1330 {
1331 PyObject *x;
1332 int version = Py_MARSHAL_VERSION;
1333 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1334 return NULL;
1335 return PyMarshal_WriteObjectToString(x, version);
1336 }
1337
1338 PyDoc_STRVAR(dumps_doc,
1339 "dumps(value[, version])\n\
1340 \n\
1341 Return the string that would be written to a file by dump(value, file).\n\
1342 The value must be a supported type. Raise a ValueError exception if\n\
1343 value has (or contains an object that has) an unsupported type.\n\
1344 \n\
1345 New in version 2.4: The version argument indicates the data format that\n\
1346 dumps should use.");
1347
1348
1349 static PyObject *
1350 marshal_loads(PyObject *self, PyObject *args)
1351 {
1352 RFILE rf;
1353 char *s;
1354 Py_ssize_t n;
1355 PyObject* result;
1356 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1357 return NULL;
1358 rf.fp = NULL;
1359 rf.ptr = s;
1360 rf.end = s + n;
1361 rf.strings = PyList_New(0);
1362 rf.depth = 0;
1363 result = read_object(&rf);
1364 Py_DECREF(rf.strings);
1365 return result;
1366 }
1367
1368 PyDoc_STRVAR(loads_doc,
1369 "loads(string)\n\
1370 \n\
1371 Convert the string to a value. If no valid value is found, raise\n\
1372 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1373 ignored.");
1374
1375 static PyMethodDef marshal_methods[] = {
1376 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1377 {"load", marshal_load, METH_O, load_doc},
1378 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1379 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1380 {NULL, NULL} /* sentinel */
1381 };
1382
1383 PyDoc_STRVAR(marshal_doc,
1384 "This module contains functions that can read and write Python values in\n\
1385 a binary format. The format is specific to Python, but independent of\n\
1386 machine architecture issues.\n\
1387 \n\
1388 Not all Python object types are supported; in general, only objects\n\
1389 whose value is independent from a particular invocation of Python can be\n\
1390 written and read by this module. The following types are supported:\n\
1391 None, integers, long integers, floating point numbers, strings, Unicode\n\
1392 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1393 should be understood that tuples, lists and dictionaries are only\n\
1394 supported as long as the values contained therein are themselves\n\
1395 supported; and recursive lists and dictionaries should not be written\n\
1396 (they will cause infinite loops).\n\
1397 \n\
1398 Variables:\n\
1399 \n\
1400 version -- indicates the format that the module uses. Version 0 is the\n\
1401 historical format, version 1 (added in Python 2.4) shares interned\n\
1402 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1403 floating point numbers. (New in version 2.4)\n\
1404 \n\
1405 Functions:\n\
1406 \n\
1407 dump() -- write value to a file\n\
1408 load() -- read value from a file\n\
1409 dumps() -- write value to a string\n\
1410 loads() -- read value from a string");
1411
1412
1413 PyMODINIT_FUNC
1414 PyMarshal_Init(void)
1415 {
1416 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1417 marshal_doc);
1418 if (mod == NULL)
1419 return;
1420 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1421 }
python git mirror