?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Fix compilation error in debug mode.
[python.git] / Python / marshal.c
blobca2f9aa10f9bb8f2f53bdbd8afcfedddcb39e1e6
1
2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
6
7 #define PY_SSIZE_T_CLEAN
8
9 #include "Python.h"
10 #include "longintrepr.h"
11 #include "code.h"
12 #include "marshal.h"
13
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
15
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
19 */
20 #define MAX_MARSHAL_STACK_DEPTH 2000
21
22 #define TYPE_NULL '0'
23 #define TYPE_NONE 'N'
24 #define TYPE_FALSE 'F'
25 #define TYPE_TRUE 'T'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
28 #define TYPE_INT 'i'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
34 #define TYPE_LONG 'l'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
39 #define TYPE_LIST '['
40 #define TYPE_DICT '{'
41 #define TYPE_CODE 'c'
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
44 #define TYPE_SET '<'
45 #define TYPE_FROZENSET '>'
46
47 #define WFERR_OK 0
48 #define WFERR_UNMARSHALLABLE 1
49 #define WFERR_NESTEDTOODEEP 2
50 #define WFERR_NOMEMORY 3
51
52 typedef struct {
53 FILE *fp;
54 int error; /* see WFERR_* values */
55 int depth;
56 /* If fp == NULL, the following are valid: */
57 PyObject *str;
58 char *ptr;
59 char *end;
60 PyObject *strings; /* dict on marshal, list on unmarshal */
61 int version;
62 } WFILE;
63
64 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
65 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
66 else w_more(c, p)
67
68 static void
69 w_more(int c, WFILE *p)
70 {
71 Py_ssize_t size, newsize;
72 if (p->str == NULL)
73 return; /* An error already occurred */
74 size = PyString_Size(p->str);
75 newsize = size + size + 1024;
76 if (newsize > 32*1024*1024) {
77 newsize = size + (size >> 3); /* 12.5% overallocation */
78 }
79 if (_PyString_Resize(&p->str, newsize) != 0) {
80 p->ptr = p->end = NULL;
81 }
82 else {
83 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
84 p->end =
85 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
86 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
87 }
88 }
89
90 static void
91 w_string(char *s, int n, WFILE *p)
92 {
93 if (p->fp != NULL) {
94 fwrite(s, 1, n, p->fp);
95 }
96 else {
97 while (--n >= 0) {
98 w_byte(*s, p);
99 s++;
100 }
101 }
102 }
103
104 static void
105 w_short(int x, WFILE *p)
106 {
107 w_byte((char)( x & 0xff), p);
108 w_byte((char)((x>> 8) & 0xff), p);
109 }
110
111 static void
112 w_long(long x, WFILE *p)
113 {
114 w_byte((char)( x & 0xff), p);
115 w_byte((char)((x>> 8) & 0xff), p);
116 w_byte((char)((x>>16) & 0xff), p);
117 w_byte((char)((x>>24) & 0xff), p);
118 }
119
120 #if SIZEOF_LONG > 4
121 static void
122 w_long64(long x, WFILE *p)
123 {
124 w_long(x, p);
125 w_long(x>>32, p);
126 }
127 #endif
128
129 /* We assume that Python longs are stored internally in base some power of
130 2**15; for the sake of portability we'll always read and write them in base
131 exactly 2**15. */
132
133 #define PyLong_MARSHAL_SHIFT 15
134 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
135 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
136 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
137 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
138 #endif
139 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
140
141 static void
142 w_PyLong(const PyLongObject *ob, WFILE *p)
143 {
144 Py_ssize_t i, j, n, l;
145 digit d;
146
147 w_byte(TYPE_LONG, p);
148 if (Py_SIZE(ob) == 0) {
149 w_long((long)0, p);
150 return;
151 }
152
153 /* set l to number of base PyLong_MARSHAL_BASE digits */
154 n = ABS(Py_SIZE(ob));
155 l = (n-1) * PyLong_MARSHAL_RATIO;
156 d = ob->ob_digit[n-1];
157 assert(d != 0); /* a PyLong is always normalized */
158 do {
159 d >>= PyLong_MARSHAL_SHIFT;
160 l++;
161 } while (d != 0);
162 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
163
164 for (i=0; i < n-1; i++) {
165 d = ob->ob_digit[i];
166 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
167 w_short(d & PyLong_MARSHAL_MASK, p);
168 d >>= PyLong_MARSHAL_SHIFT;
169 }
170 assert (d == 0);
171 }
172 d = ob->ob_digit[n-1];
173 do {
174 w_short(d & PyLong_MARSHAL_MASK, p);
175 d >>= PyLong_MARSHAL_SHIFT;
176 } while (d != 0);
177 }
178
179 static void
180 w_object(PyObject *v, WFILE *p)
181 {
182 Py_ssize_t i, n;
183
184 p->depth++;
185
186 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
187 p->error = WFERR_NESTEDTOODEEP;
188 }
189 else if (v == NULL) {
190 w_byte(TYPE_NULL, p);
191 }
192 else if (v == Py_None) {
193 w_byte(TYPE_NONE, p);
194 }
195 else if (v == PyExc_StopIteration) {
196 w_byte(TYPE_STOPITER, p);
197 }
198 else if (v == Py_Ellipsis) {
199 w_byte(TYPE_ELLIPSIS, p);
200 }
201 else if (v == Py_False) {
202 w_byte(TYPE_FALSE, p);
203 }
204 else if (v == Py_True) {
205 w_byte(TYPE_TRUE, p);
206 }
207 else if (PyInt_CheckExact(v)) {
208 long x = PyInt_AS_LONG((PyIntObject *)v);
209 #if SIZEOF_LONG > 4
210 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
211 if (y && y != -1) {
212 w_byte(TYPE_INT64, p);
213 w_long64(x, p);
214 }
215 else
216 #endif
217 {
218 w_byte(TYPE_INT, p);
219 w_long(x, p);
220 }
221 }
222 else if (PyLong_CheckExact(v)) {
223 PyLongObject *ob = (PyLongObject *)v;
224 w_PyLong(ob, p);
225 }
226 else if (PyFloat_CheckExact(v)) {
227 if (p->version > 1) {
228 unsigned char buf[8];
229 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
230 buf, 1) < 0) {
231 p->error = WFERR_UNMARSHALLABLE;
232 return;
233 }
234 w_byte(TYPE_BINARY_FLOAT, p);
235 w_string((char*)buf, 8, p);
236 }
237 else {
238 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
239 'g', 17, 0, NULL);
240 if (!buf) {
241 p->error = WFERR_NOMEMORY;
242 return;
243 }
244 n = strlen(buf);
245 w_byte(TYPE_FLOAT, p);
246 w_byte((int)n, p);
247 w_string(buf, (int)n, p);
248 PyMem_Free(buf);
249 }
250 }
251 #ifndef WITHOUT_COMPLEX
252 else if (PyComplex_CheckExact(v)) {
253 if (p->version > 1) {
254 unsigned char buf[8];
255 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
256 buf, 1) < 0) {
257 p->error = WFERR_UNMARSHALLABLE;
258 return;
259 }
260 w_byte(TYPE_BINARY_COMPLEX, p);
261 w_string((char*)buf, 8, p);
262 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
263 buf, 1) < 0) {
264 p->error = WFERR_UNMARSHALLABLE;
265 return;
266 }
267 w_string((char*)buf, 8, p);
268 }
269 else {
270 char *buf;
271 w_byte(TYPE_COMPLEX, p);
272 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
273 'g', 17, 0, NULL);
274 if (!buf) {
275 p->error = WFERR_NOMEMORY;
276 return;
277 }
278 n = strlen(buf);
279 w_byte((int)n, p);
280 w_string(buf, (int)n, p);
281 PyMem_Free(buf);
282 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
283 'g', 17, 0, NULL);
284 if (!buf) {
285 p->error = WFERR_NOMEMORY;
286 return;
287 }
288 n = strlen(buf);
289 w_byte((int)n, p);
290 w_string(buf, (int)n, p);
291 PyMem_Free(buf);
292 }
293 }
294 #endif
295 else if (PyString_CheckExact(v)) {
296 if (p->strings && PyString_CHECK_INTERNED(v)) {
297 PyObject *o = PyDict_GetItem(p->strings, v);
298 if (o) {
299 long w = PyInt_AsLong(o);
300 w_byte(TYPE_STRINGREF, p);
301 w_long(w, p);
302 goto exit;
303 }
304 else {
305 int ok;
306 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
307 ok = o &&
308 PyDict_SetItem(p->strings, v, o) >= 0;
309 Py_XDECREF(o);
310 if (!ok) {
311 p->depth--;
312 p->error = WFERR_UNMARSHALLABLE;
313 return;
314 }
315 w_byte(TYPE_INTERNED, p);
316 }
317 }
318 else {
319 w_byte(TYPE_STRING, p);
320 }
321 n = PyString_GET_SIZE(v);
322 if (n > INT_MAX) {
323 /* huge strings are not supported */
324 p->depth--;
325 p->error = WFERR_UNMARSHALLABLE;
326 return;
327 }
328 w_long((long)n, p);
329 w_string(PyString_AS_STRING(v), (int)n, p);
330 }
331 #ifdef Py_USING_UNICODE
332 else if (PyUnicode_CheckExact(v)) {
333 PyObject *utf8;
334 utf8 = PyUnicode_AsUTF8String(v);
335 if (utf8 == NULL) {
336 p->depth--;
337 p->error = WFERR_UNMARSHALLABLE;
338 return;
339 }
340 w_byte(TYPE_UNICODE, p);
341 n = PyString_GET_SIZE(utf8);
342 if (n > INT_MAX) {
343 p->depth--;
344 p->error = WFERR_UNMARSHALLABLE;
345 return;
346 }
347 w_long((long)n, p);
348 w_string(PyString_AS_STRING(utf8), (int)n, p);
349 Py_DECREF(utf8);
350 }
351 #endif
352 else if (PyTuple_CheckExact(v)) {
353 w_byte(TYPE_TUPLE, p);
354 n = PyTuple_Size(v);
355 w_long((long)n, p);
356 for (i = 0; i < n; i++) {
357 w_object(PyTuple_GET_ITEM(v, i), p);
358 }
359 }
360 else if (PyList_CheckExact(v)) {
361 w_byte(TYPE_LIST, p);
362 n = PyList_GET_SIZE(v);
363 w_long((long)n, p);
364 for (i = 0; i < n; i++) {
365 w_object(PyList_GET_ITEM(v, i), p);
366 }
367 }
368 else if (PyDict_CheckExact(v)) {
369 Py_ssize_t pos;
370 PyObject *key, *value;
371 w_byte(TYPE_DICT, p);
372 /* This one is NULL object terminated! */
373 pos = 0;
374 while (PyDict_Next(v, &pos, &key, &value)) {
375 w_object(key, p);
376 w_object(value, p);
377 }
378 w_object((PyObject *)NULL, p);
379 }
380 else if (PyAnySet_CheckExact(v)) {
381 PyObject *value, *it;
382
383 if (PyObject_TypeCheck(v, &PySet_Type))
384 w_byte(TYPE_SET, p);
385 else
386 w_byte(TYPE_FROZENSET, p);
387 n = PyObject_Size(v);
388 if (n == -1) {
389 p->depth--;
390 p->error = WFERR_UNMARSHALLABLE;
391 return;
392 }
393 w_long((long)n, p);
394 it = PyObject_GetIter(v);
395 if (it == NULL) {
396 p->depth--;
397 p->error = WFERR_UNMARSHALLABLE;
398 return;
399 }
400 while ((value = PyIter_Next(it)) != NULL) {
401 w_object(value, p);
402 Py_DECREF(value);
403 }
404 Py_DECREF(it);
405 if (PyErr_Occurred()) {
406 p->depth--;
407 p->error = WFERR_UNMARSHALLABLE;
408 return;
409 }
410 }
411 else if (PyCode_Check(v)) {
412 PyCodeObject *co = (PyCodeObject *)v;
413 w_byte(TYPE_CODE, p);
414 w_long(co->co_argcount, p);
415 w_long(co->co_nlocals, p);
416 w_long(co->co_stacksize, p);
417 w_long(co->co_flags, p);
418 w_object(co->co_code, p);
419 w_object(co->co_consts, p);
420 w_object(co->co_names, p);
421 w_object(co->co_varnames, p);
422 w_object(co->co_freevars, p);
423 w_object(co->co_cellvars, p);
424 w_object(co->co_filename, p);
425 w_object(co->co_name, p);
426 w_long(co->co_firstlineno, p);
427 w_object(co->co_lnotab, p);
428 }
429 else if (PyObject_CheckReadBuffer(v)) {
430 /* Write unknown buffer-style objects as a string */
431 char *s;
432 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
433 w_byte(TYPE_STRING, p);
434 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
435 if (n > INT_MAX) {
436 p->depth--;
437 p->error = WFERR_UNMARSHALLABLE;
438 return;
439 }
440 w_long((long)n, p);
441 w_string(s, (int)n, p);
442 }
443 else {
444 w_byte(TYPE_UNKNOWN, p);
445 p->error = WFERR_UNMARSHALLABLE;
446 }
447 exit:
448 p->depth--;
449 }
450
451 /* version currently has no effect for writing longs. */
452 void
453 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
454 {
455 WFILE wf;
456 wf.fp = fp;
457 wf.error = WFERR_OK;
458 wf.depth = 0;
459 wf.strings = NULL;
460 wf.version = version;
461 w_long(x, &wf);
462 }
463
464 void
465 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
466 {
467 WFILE wf;
468 wf.fp = fp;
469 wf.error = WFERR_OK;
470 wf.depth = 0;
471 wf.strings = (version > 0) ? PyDict_New() : NULL;
472 wf.version = version;
473 w_object(x, &wf);
474 Py_XDECREF(wf.strings);
475 }
476
477 typedef WFILE RFILE; /* Same struct with different invariants */
478
479 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
480
481 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
482
483 static int
484 r_string(char *s, int n, RFILE *p)
485 {
486 if (p->fp != NULL)
487 /* The result fits into int because it must be <=n. */
488 return (int)fread(s, 1, n, p->fp);
489 if (p->end - p->ptr < n)
490 n = (int)(p->end - p->ptr);
491 memcpy(s, p->ptr, n);
492 p->ptr += n;
493 return n;
494 }
495
496 static int
497 r_short(RFILE *p)
498 {
499 register short x;
500 x = r_byte(p);
501 x |= r_byte(p) << 8;
502 /* Sign-extension, in case short greater than 16 bits */
503 x |= -(x & 0x8000);
504 return x;
505 }
506
507 static long
508 r_long(RFILE *p)
509 {
510 register long x;
511 register FILE *fp = p->fp;
512 if (fp) {
513 x = getc(fp);
514 x |= (long)getc(fp) << 8;
515 x |= (long)getc(fp) << 16;
516 x |= (long)getc(fp) << 24;
517 }
518 else {
519 x = rs_byte(p);
520 x |= (long)rs_byte(p) << 8;
521 x |= (long)rs_byte(p) << 16;
522 x |= (long)rs_byte(p) << 24;
523 }
524 #if SIZEOF_LONG > 4
525 /* Sign extension for 64-bit machines */
526 x |= -(x & 0x80000000L);
527 #endif
528 return x;
529 }
530
531 /* r_long64 deals with the TYPE_INT64 code. On a machine with
532 sizeof(long) > 4, it returns a Python int object, else a Python long
533 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
534 so there's no inefficiency here in returning a PyLong on 32-bit boxes
535 for everything written via TYPE_INT64 (i.e., if an int is written via
536 TYPE_INT64, it *needs* more than 32 bits).
537 */
538 static PyObject *
539 r_long64(RFILE *p)
540 {
541 long lo4 = r_long(p);
542 long hi4 = r_long(p);
543 #if SIZEOF_LONG > 4
544 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
545 return PyInt_FromLong(x);
546 #else
547 unsigned char buf[8];
548 int one = 1;
549 int is_little_endian = (int)*(char*)&one;
550 if (is_little_endian) {
551 memcpy(buf, &lo4, 4);
552 memcpy(buf+4, &hi4, 4);
553 }
554 else {
555 memcpy(buf, &hi4, 4);
556 memcpy(buf+4, &lo4, 4);
557 }
558 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
559 #endif
560 }
561
562 static PyObject *
563 r_PyLong(RFILE *p)
564 {
565 PyLongObject *ob;
566 int size, i, j, md, shorts_in_top_digit;
567 long n;
568 digit d;
569
570 n = r_long(p);
571 if (n == 0)
572 return (PyObject *)_PyLong_New(0);
573 if (n < -INT_MAX || n > INT_MAX) {
574 PyErr_SetString(PyExc_ValueError,
575 "bad marshal data (long size out of range)");
576 return NULL;
577 }
578
579 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
580 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
581 ob = _PyLong_New(size);
582 if (ob == NULL)
583 return NULL;
584 Py_SIZE(ob) = n > 0 ? size : -size;
585
586 for (i = 0; i < size-1; i++) {
587 d = 0;
588 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
589 md = r_short(p);
590 if (md < 0 || md > PyLong_MARSHAL_BASE)
591 goto bad_digit;
592 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
593 }
594 ob->ob_digit[i] = d;
595 }
596 d = 0;
597 for (j=0; j < shorts_in_top_digit; j++) {
598 md = r_short(p);
599 if (md < 0 || md > PyLong_MARSHAL_BASE)
600 goto bad_digit;
601 /* topmost marshal digit should be nonzero */
602 if (md == 0 && j == shorts_in_top_digit - 1) {
603 Py_DECREF(ob);
604 PyErr_SetString(PyExc_ValueError,
605 "bad marshal data (unnormalized long data)");
606 return NULL;
607 }
608 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
609 }
610 /* top digit should be nonzero, else the resulting PyLong won't be
611 normalized */
612 ob->ob_digit[size-1] = d;
613 return (PyObject *)ob;
614 bad_digit:
615 Py_DECREF(ob);
616 PyErr_SetString(PyExc_ValueError,
617 "bad marshal data (digit out of range in long)");
618 return NULL;
619 }
620
621
622 static PyObject *
623 r_object(RFILE *p)
624 {
625 /* NULL is a valid return value, it does not necessarily means that
626 an exception is set. */
627 PyObject *v, *v2;
628 long i, n;
629 int type = r_byte(p);
630 PyObject *retval;
631
632 p->depth++;
633
634 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
635 p->depth--;
636 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
637 return NULL;
638 }
639
640 switch (type) {
641
642 case EOF:
643 PyErr_SetString(PyExc_EOFError,
644 "EOF read where object expected");
645 retval = NULL;
646 break;
647
648 case TYPE_NULL:
649 retval = NULL;
650 break;
651
652 case TYPE_NONE:
653 Py_INCREF(Py_None);
654 retval = Py_None;
655 break;
656
657 case TYPE_STOPITER:
658 Py_INCREF(PyExc_StopIteration);
659 retval = PyExc_StopIteration;
660 break;
661
662 case TYPE_ELLIPSIS:
663 Py_INCREF(Py_Ellipsis);
664 retval = Py_Ellipsis;
665 break;
666
667 case TYPE_FALSE:
668 Py_INCREF(Py_False);
669 retval = Py_False;
670 break;
671
672 case TYPE_TRUE:
673 Py_INCREF(Py_True);
674 retval = Py_True;
675 break;
676
677 case TYPE_INT:
678 retval = PyInt_FromLong(r_long(p));
679 break;
680
681 case TYPE_INT64:
682 retval = r_long64(p);
683 break;
684
685 case TYPE_LONG:
686 retval = r_PyLong(p);
687 break;
688
689 case TYPE_FLOAT:
690 {
691 char buf[256];
692 double dx;
693 n = r_byte(p);
694 if (n == EOF || r_string(buf, (int)n, p) != n) {
695 PyErr_SetString(PyExc_EOFError,
696 "EOF read where object expected");
697 retval = NULL;
698 break;
699 }
700 buf[n] = '\0';
701 retval = NULL;
702 PyFPE_START_PROTECT("atof", break)
703 dx = PyOS_ascii_atof(buf);
704 PyFPE_END_PROTECT(dx)
705 retval = PyFloat_FromDouble(dx);
706 break;
707 }
708
709 case TYPE_BINARY_FLOAT:
710 {
711 unsigned char buf[8];
712 double x;
713 if (r_string((char*)buf, 8, p) != 8) {
714 PyErr_SetString(PyExc_EOFError,
715 "EOF read where object expected");
716 retval = NULL;
717 break;
718 }
719 x = _PyFloat_Unpack8(buf, 1);
720 if (x == -1.0 && PyErr_Occurred()) {
721 retval = NULL;
722 break;
723 }
724 retval = PyFloat_FromDouble(x);
725 break;
726 }
727
728 #ifndef WITHOUT_COMPLEX
729 case TYPE_COMPLEX:
730 {
731 char buf[256];
732 Py_complex c;
733 n = r_byte(p);
734 if (n == EOF || r_string(buf, (int)n, p) != n) {
735 PyErr_SetString(PyExc_EOFError,
736 "EOF read where object expected");
737 retval = NULL;
738 break;
739 }
740 buf[n] = '\0';
741 retval = NULL;
742 PyFPE_START_PROTECT("atof", break;)
743 c.real = PyOS_ascii_atof(buf);
744 PyFPE_END_PROTECT(c)
745 n = r_byte(p);
746 if (n == EOF || r_string(buf, (int)n, p) != n) {
747 PyErr_SetString(PyExc_EOFError,
748 "EOF read where object expected");
749 retval = NULL;
750 break;
751 }
752 buf[n] = '\0';
753 PyFPE_START_PROTECT("atof", break)
754 c.imag = PyOS_ascii_atof(buf);
755 PyFPE_END_PROTECT(c)
756 retval = PyComplex_FromCComplex(c);
757 break;
758 }
759
760 case TYPE_BINARY_COMPLEX:
761 {
762 unsigned char buf[8];
763 Py_complex c;
764 if (r_string((char*)buf, 8, p) != 8) {
765 PyErr_SetString(PyExc_EOFError,
766 "EOF read where object expected");
767 retval = NULL;
768 break;
769 }
770 c.real = _PyFloat_Unpack8(buf, 1);
771 if (c.real == -1.0 && PyErr_Occurred()) {
772 retval = NULL;
773 break;
774 }
775 if (r_string((char*)buf, 8, p) != 8) {
776 PyErr_SetString(PyExc_EOFError,
777 "EOF read where object expected");
778 retval = NULL;
779 break;
780 }
781 c.imag = _PyFloat_Unpack8(buf, 1);
782 if (c.imag == -1.0 && PyErr_Occurred()) {
783 retval = NULL;
784 break;
785 }
786 retval = PyComplex_FromCComplex(c);
787 break;
788 }
789 #endif
790
791 case TYPE_INTERNED:
792 case TYPE_STRING:
793 n = r_long(p);
794 if (n < 0 || n > INT_MAX) {
795 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
796 retval = NULL;
797 break;
798 }
799 v = PyString_FromStringAndSize((char *)NULL, n);
800 if (v == NULL) {
801 retval = NULL;
802 break;
803 }
804 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
805 Py_DECREF(v);
806 PyErr_SetString(PyExc_EOFError,
807 "EOF read where object expected");
808 retval = NULL;
809 break;
810 }
811 if (type == TYPE_INTERNED) {
812 PyString_InternInPlace(&v);
813 if (PyList_Append(p->strings, v) < 0) {
814 retval = NULL;
815 break;
816 }
817 }
818 retval = v;
819 break;
820
821 case TYPE_STRINGREF:
822 n = r_long(p);
823 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
824 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)");
825 retval = NULL;
826 break;
827 }
828 v = PyList_GET_ITEM(p->strings, n);
829 Py_INCREF(v);
830 retval = v;
831 break;
832
833 #ifdef Py_USING_UNICODE
834 case TYPE_UNICODE:
835 {
836 char *buffer;
837
838 n = r_long(p);
839 if (n < 0 || n > INT_MAX) {
840 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
841 retval = NULL;
842 break;
843 }
844 buffer = PyMem_NEW(char, n);
845 if (buffer == NULL) {
846 retval = PyErr_NoMemory();
847 break;
848 }
849 if (r_string(buffer, (int)n, p) != n) {
850 PyMem_DEL(buffer);
851 PyErr_SetString(PyExc_EOFError,
852 "EOF read where object expected");
853 retval = NULL;
854 break;
855 }
856 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
857 PyMem_DEL(buffer);
858 retval = v;
859 break;
860 }
861 #endif
862
863 case TYPE_TUPLE:
864 n = r_long(p);
865 if (n < 0 || n > INT_MAX) {
866 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
867 retval = NULL;
868 break;
869 }
870 v = PyTuple_New((int)n);
871 if (v == NULL) {
872 retval = NULL;
873 break;
874 }
875 for (i = 0; i < n; i++) {
876 v2 = r_object(p);
877 if ( v2 == NULL ) {
878 if (!PyErr_Occurred())
879 PyErr_SetString(PyExc_TypeError,
880 "NULL object in marshal data for tuple");
881 Py_DECREF(v);
882 v = NULL;
883 break;
884 }
885 PyTuple_SET_ITEM(v, (int)i, v2);
886 }
887 retval = v;
888 break;
889
890 case TYPE_LIST:
891 n = r_long(p);
892 if (n < 0 || n > INT_MAX) {
893 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
894 retval = NULL;
895 break;
896 }
897 v = PyList_New((int)n);
898 if (v == NULL) {
899 retval = NULL;
900 break;
901 }
902 for (i = 0; i < n; i++) {
903 v2 = r_object(p);
904 if ( v2 == NULL ) {
905 if (!PyErr_Occurred())
906 PyErr_SetString(PyExc_TypeError,
907 "NULL object in marshal data for list");
908 Py_DECREF(v);
909 v = NULL;
910 break;
911 }
912 PyList_SET_ITEM(v, (int)i, v2);
913 }
914 retval = v;
915 break;
916
917 case TYPE_DICT:
918 v = PyDict_New();
919 if (v == NULL) {
920 retval = NULL;
921 break;
922 }
923 for (;;) {
924 PyObject *key, *val;
925 key = r_object(p);
926 if (key == NULL)
927 break;
928 val = r_object(p);
929 if (val != NULL)
930 PyDict_SetItem(v, key, val);
931 Py_DECREF(key);
932 Py_XDECREF(val);
933 }
934 if (PyErr_Occurred()) {
935 Py_DECREF(v);
936 v = NULL;
937 }
938 retval = v;
939 break;
940
941 case TYPE_SET:
942 case TYPE_FROZENSET:
943 n = r_long(p);
944 if (n < 0 || n > INT_MAX) {
945 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
946 retval = NULL;
947 break;
948 }
949 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
950 if (v == NULL) {
951 retval = NULL;
952 break;
953 }
954 for (i = 0; i < n; i++) {
955 v2 = r_object(p);
956 if ( v2 == NULL ) {
957 if (!PyErr_Occurred())
958 PyErr_SetString(PyExc_TypeError,
959 "NULL object in marshal data for set");
960 Py_DECREF(v);
961 v = NULL;
962 break;
963 }
964 if (PySet_Add(v, v2) == -1) {
965 Py_DECREF(v);
966 Py_DECREF(v2);
967 v = NULL;
968 break;
969 }
970 Py_DECREF(v2);
971 }
972 retval = v;
973 break;
974
975 case TYPE_CODE:
976 if (PyEval_GetRestricted()) {
977 PyErr_SetString(PyExc_RuntimeError,
978 "cannot unmarshal code objects in "
979 "restricted execution mode");
980 retval = NULL;
981 break;
982 }
983 else {
984 int argcount;
985 int nlocals;
986 int stacksize;
987 int flags;
988 PyObject *code = NULL;
989 PyObject *consts = NULL;
990 PyObject *names = NULL;
991 PyObject *varnames = NULL;
992 PyObject *freevars = NULL;
993 PyObject *cellvars = NULL;
994 PyObject *filename = NULL;
995 PyObject *name = NULL;
996 int firstlineno;
997 PyObject *lnotab = NULL;
998
999 v = NULL;
1000
1001 /* XXX ignore long->int overflows for now */
1002 argcount = (int)r_long(p);
1003 nlocals = (int)r_long(p);
1004 stacksize = (int)r_long(p);
1005 flags = (int)r_long(p);
1006 code = r_object(p);
1007 if (code == NULL)
1008 goto code_error;
1009 consts = r_object(p);
1010 if (consts == NULL)
1011 goto code_error;
1012 names = r_object(p);
1013 if (names == NULL)
1014 goto code_error;
1015 varnames = r_object(p);
1016 if (varnames == NULL)
1017 goto code_error;
1018 freevars = r_object(p);
1019 if (freevars == NULL)
1020 goto code_error;
1021 cellvars = r_object(p);
1022 if (cellvars == NULL)
1023 goto code_error;
1024 filename = r_object(p);
1025 if (filename == NULL)
1026 goto code_error;
1027 name = r_object(p);
1028 if (name == NULL)
1029 goto code_error;
1030 firstlineno = (int)r_long(p);
1031 lnotab = r_object(p);
1032 if (lnotab == NULL)
1033 goto code_error;
1034
1035 v = (PyObject *) PyCode_New(
1036 argcount, nlocals, stacksize, flags,
1037 code, consts, names, varnames,
1038 freevars, cellvars, filename, name,
1039 firstlineno, lnotab);
1040
1041 code_error:
1042 Py_XDECREF(code);
1043 Py_XDECREF(consts);
1044 Py_XDECREF(names);
1045 Py_XDECREF(varnames);
1046 Py_XDECREF(freevars);
1047 Py_XDECREF(cellvars);
1048 Py_XDECREF(filename);
1049 Py_XDECREF(name);
1050 Py_XDECREF(lnotab);
1051
1052 }
1053 retval = v;
1054 break;
1055
1056 default:
1057 /* Bogus data got written, which isn't ideal.
1058 This will let you keep working and recover. */
1059 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1060 retval = NULL;
1061 break;
1062
1063 }
1064 p->depth--;
1065 return retval;
1066 }
1067
1068 static PyObject *
1069 read_object(RFILE *p)
1070 {
1071 PyObject *v;
1072 if (PyErr_Occurred()) {
1073 fprintf(stderr, "XXX readobject called with exception set\n");
1074 return NULL;
1075 }
1076 v = r_object(p);
1077 if (v == NULL && !PyErr_Occurred())
1078 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1079 return v;
1080 }
1081
1082 int
1083 PyMarshal_ReadShortFromFile(FILE *fp)
1084 {
1085 RFILE rf;
1086 assert(fp);
1087 rf.fp = fp;
1088 rf.strings = NULL;
1089 rf.end = rf.ptr = NULL;
1090 return r_short(&rf);
1091 }
1092
1093 long
1094 PyMarshal_ReadLongFromFile(FILE *fp)
1095 {
1096 RFILE rf;
1097 rf.fp = fp;
1098 rf.strings = NULL;
1099 rf.ptr = rf.end = NULL;
1100 return r_long(&rf);
1101 }
1102
1103 #ifdef HAVE_FSTAT
1104 /* Return size of file in bytes; < 0 if unknown. */
1105 static off_t
1106 getfilesize(FILE *fp)
1107 {
1108 struct stat st;
1109 if (fstat(fileno(fp), &st) != 0)
1110 return -1;
1111 else
1112 return st.st_size;
1113 }
1114 #endif
1115
1116 /* If we can get the size of the file up-front, and it's reasonably small,
1117 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1118 * than reading a byte at a time from file; speeds .pyc imports.
1119 * CAUTION: since this may read the entire remainder of the file, don't
1120 * call it unless you know you're done with the file.
1121 */
1122 PyObject *
1123 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1124 {
1125 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1126 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1127 */
1128 #define SMALL_FILE_LIMIT (1L << 14)
1129 #define REASONABLE_FILE_LIMIT (1L << 18)
1130 #ifdef HAVE_FSTAT
1131 off_t filesize;
1132 #endif
1133 #ifdef HAVE_FSTAT
1134 filesize = getfilesize(fp);
1135 if (filesize > 0) {
1136 char buf[SMALL_FILE_LIMIT];
1137 char* pBuf = NULL;
1138 if (filesize <= SMALL_FILE_LIMIT)
1139 pBuf = buf;
1140 else if (filesize <= REASONABLE_FILE_LIMIT)
1141 pBuf = (char *)PyMem_MALLOC(filesize);
1142 if (pBuf != NULL) {
1143 PyObject* v;
1144 size_t n;
1145 /* filesize must fit into an int, because it
1146 is smaller than REASONABLE_FILE_LIMIT */
1147 n = fread(pBuf, 1, (int)filesize, fp);
1148 v = PyMarshal_ReadObjectFromString(pBuf, n);
1149 if (pBuf != buf)
1150 PyMem_FREE(pBuf);
1151 return v;
1152 }
1153
1154 }
1155 #endif
1156 /* We don't have fstat, or we do but the file is larger than
1157 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1158 */
1159 return PyMarshal_ReadObjectFromFile(fp);
1160
1161 #undef SMALL_FILE_LIMIT
1162 #undef REASONABLE_FILE_LIMIT
1163 }
1164
1165 PyObject *
1166 PyMarshal_ReadObjectFromFile(FILE *fp)
1167 {
1168 RFILE rf;
1169 PyObject *result;
1170 rf.fp = fp;
1171 rf.strings = PyList_New(0);
1172 rf.depth = 0;
1173 rf.ptr = rf.end = NULL;
1174 result = r_object(&rf);
1175 Py_DECREF(rf.strings);
1176 return result;
1177 }
1178
1179 PyObject *
1180 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1181 {
1182 RFILE rf;
1183 PyObject *result;
1184 rf.fp = NULL;
1185 rf.ptr = str;
1186 rf.end = str + len;
1187 rf.strings = PyList_New(0);
1188 rf.depth = 0;
1189 result = r_object(&rf);
1190 Py_DECREF(rf.strings);
1191 return result;
1192 }
1193
1194 static void
1195 set_error(int error)
1196 {
1197 switch (error) {
1198 case WFERR_NOMEMORY:
1199 PyErr_NoMemory();
1200 break;
1201 case WFERR_UNMARSHALLABLE:
1202 PyErr_SetString(PyExc_ValueError, "unmarshallable object");
1203 break;
1204 case WFERR_NESTEDTOODEEP:
1205 default:
1206 PyErr_SetString(PyExc_ValueError,
1207 "object too deeply nested to marshal");
1208 break;
1209 }
1210 }
1211
1212 PyObject *
1213 PyMarshal_WriteObjectToString(PyObject *x, int version)
1214 {
1215 WFILE wf;
1216 wf.fp = NULL;
1217 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1218 if (wf.str == NULL)
1219 return NULL;
1220 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1221 wf.end = wf.ptr + PyString_Size(wf.str);
1222 wf.error = WFERR_OK;
1223 wf.depth = 0;
1224 wf.version = version;
1225 wf.strings = (version > 0) ? PyDict_New() : NULL;
1226 w_object(x, &wf);
1227 Py_XDECREF(wf.strings);
1228 if (wf.str != NULL) {
1229 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1230 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1231 Py_DECREF(wf.str);
1232 PyErr_SetString(PyExc_OverflowError,
1233 "too much marshall data for a string");
1234 return NULL;
1235 }
1236 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1237 }
1238 if (wf.error != WFERR_OK) {
1239 Py_XDECREF(wf.str);
1240 set_error(wf.error);
1241 return NULL;
1242 }
1243 return wf.str;
1244 }
1245
1246 /* And an interface for Python programs... */
1247
1248 static PyObject *
1249 marshal_dump(PyObject *self, PyObject *args)
1250 {
1251 WFILE wf;
1252 PyObject *x;
1253 PyObject *f;
1254 int version = Py_MARSHAL_VERSION;
1255 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1256 return NULL;
1257 if (!PyFile_Check(f)) {
1258 PyErr_SetString(PyExc_TypeError,
1259 "marshal.dump() 2nd arg must be file");
1260 return NULL;
1261 }
1262 wf.fp = PyFile_AsFile(f);
1263 wf.str = NULL;
1264 wf.ptr = wf.end = NULL;
1265 wf.error = WFERR_OK;
1266 wf.depth = 0;
1267 wf.strings = (version > 0) ? PyDict_New() : 0;
1268 wf.version = version;
1269 w_object(x, &wf);
1270 Py_XDECREF(wf.strings);
1271 if (wf.error != WFERR_OK) {
1272 set_error(wf.error);
1273 return NULL;
1274 }
1275 Py_INCREF(Py_None);
1276 return Py_None;
1277 }
1278
1279 PyDoc_STRVAR(dump_doc,
1280 "dump(value, file[, version])\n\
1281 \n\
1282 Write the value on the open file. The value must be a supported type.\n\
1283 The file must be an open file object such as sys.stdout or returned by\n\
1284 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1285 \n\
1286 If the value has (or contains an object that has) an unsupported type, a\n\
1287 ValueError exception is raised — but garbage data will also be written\n\
1288 to the file. The object will not be properly read back by load()\n\
1289 \n\
1290 New in version 2.4: The version argument indicates the data format that\n\
1291 dump should use.");
1292
1293 static PyObject *
1294 marshal_load(PyObject *self, PyObject *f)
1295 {
1296 RFILE rf;
1297 PyObject *result;
1298 if (!PyFile_Check(f)) {
1299 PyErr_SetString(PyExc_TypeError,
1300 "marshal.load() arg must be file");
1301 return NULL;
1302 }
1303 rf.fp = PyFile_AsFile(f);
1304 rf.strings = PyList_New(0);
1305 rf.depth = 0;
1306 result = read_object(&rf);
1307 Py_DECREF(rf.strings);
1308 return result;
1309 }
1310
1311 PyDoc_STRVAR(load_doc,
1312 "load(file)\n\
1313 \n\
1314 Read one value from the open file and return it. If no valid value is\n\
1315 read (e.g. because the data has a different Python version’s\n\
1316 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1317 The file must be an open file object opened in binary mode ('rb' or\n\
1318 'r+b').\n\
1319 \n\
1320 Note: If an object containing an unsupported type was marshalled with\n\
1321 dump(), load() will substitute None for the unmarshallable type.");
1322
1323
1324 static PyObject *
1325 marshal_dumps(PyObject *self, PyObject *args)
1326 {
1327 PyObject *x;
1328 int version = Py_MARSHAL_VERSION;
1329 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1330 return NULL;
1331 return PyMarshal_WriteObjectToString(x, version);
1332 }
1333
1334 PyDoc_STRVAR(dumps_doc,
1335 "dumps(value[, version])\n\
1336 \n\
1337 Return the string that would be written to a file by dump(value, file).\n\
1338 The value must be a supported type. Raise a ValueError exception if\n\
1339 value has (or contains an object that has) an unsupported type.\n\
1340 \n\
1341 New in version 2.4: The version argument indicates the data format that\n\
1342 dumps should use.");
1343
1344
1345 static PyObject *
1346 marshal_loads(PyObject *self, PyObject *args)
1347 {
1348 RFILE rf;
1349 char *s;
1350 Py_ssize_t n;
1351 PyObject* result;
1352 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1353 return NULL;
1354 rf.fp = NULL;
1355 rf.ptr = s;
1356 rf.end = s + n;
1357 rf.strings = PyList_New(0);
1358 rf.depth = 0;
1359 result = read_object(&rf);
1360 Py_DECREF(rf.strings);
1361 return result;
1362 }
1363
1364 PyDoc_STRVAR(loads_doc,
1365 "loads(string)\n\
1366 \n\
1367 Convert the string to a value. If no valid value is found, raise\n\
1368 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1369 ignored.");
1370
1371 static PyMethodDef marshal_methods[] = {
1372 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1373 {"load", marshal_load, METH_O, load_doc},
1374 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1375 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1376 {NULL, NULL} /* sentinel */
1377 };
1378
1379 PyDoc_STRVAR(marshal_doc,
1380 "This module contains functions that can read and write Python values in\n\
1381 a binary format. The format is specific to Python, but independent of\n\
1382 machine architecture issues.\n\
1383 \n\
1384 Not all Python object types are supported; in general, only objects\n\
1385 whose value is independent from a particular invocation of Python can be\n\
1386 written and read by this module. The following types are supported:\n\
1387 None, integers, long integers, floating point numbers, strings, Unicode\n\
1388 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1389 should be understood that tuples, lists and dictionaries are only\n\
1390 supported as long as the values contained therein are themselves\n\
1391 supported; and recursive lists and dictionaries should not be written\n\
1392 (they will cause infinite loops).\n\
1393 \n\
1394 Variables:\n\
1395 \n\
1396 version -- indicates the format that the module uses. Version 0 is the\n\
1397 historical format, version 1 (added in Python 2.4) shares interned\n\
1398 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1399 floating point numbers. (New in version 2.4)\n\
1400 \n\
1401 Functions:\n\
1402 \n\
1403 dump() -- write value to a file\n\
1404 load() -- read value from a file\n\
1405 dumps() -- write value to a string\n\
1406 loads() -- read value from a string");
1407
1408
1409 PyMODINIT_FUNC
1410 PyMarshal_Init(void)
1411 {
1412 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1413 marshal_doc);
1414 if (mod == NULL)
1415 return;
1416 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1417 }
python git mirror