?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Fix transient refleaks in test_docxmlrpc.
[python.git] / Python / marshal.c
blob865a0c88715a1ee994534eca93b0707e6e9eba65
1
2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
6
7 #define PY_SSIZE_T_CLEAN
8
9 #include "Python.h"
10 #include "longintrepr.h"
11 #include "code.h"
12 #include "marshal.h"
13
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
15
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
19 */
20 #define MAX_MARSHAL_STACK_DEPTH 2000
21
22 #define TYPE_NULL '0'
23 #define TYPE_NONE 'N'
24 #define TYPE_FALSE 'F'
25 #define TYPE_TRUE 'T'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
28 #define TYPE_INT 'i'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
34 #define TYPE_LONG 'l'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
39 #define TYPE_LIST '['
40 #define TYPE_DICT '{'
41 #define TYPE_CODE 'c'
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
44 #define TYPE_SET '<'
45 #define TYPE_FROZENSET '>'
46
47 #define WFERR_OK 0
48 #define WFERR_UNMARSHALLABLE 1
49 #define WFERR_NESTEDTOODEEP 2
50 #define WFERR_NOMEMORY 3
51
52 typedef struct {
53 FILE *fp;
54 int error; /* see WFERR_* values */
55 int depth;
56 /* If fp == NULL, the following are valid: */
57 PyObject *str;
58 char *ptr;
59 char *end;
60 PyObject *strings; /* dict on marshal, list on unmarshal */
61 int version;
62 } WFILE;
63
64 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
65 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
66 else w_more(c, p)
67
68 static void
69 w_more(int c, WFILE *p)
70 {
71 Py_ssize_t size, newsize;
72 if (p->str == NULL)
73 return; /* An error already occurred */
74 size = PyString_Size(p->str);
75 newsize = size + size + 1024;
76 if (newsize > 32*1024*1024) {
77 newsize = size + (size >> 3); /* 12.5% overallocation */
78 }
79 if (_PyString_Resize(&p->str, newsize) != 0) {
80 p->ptr = p->end = NULL;
81 }
82 else {
83 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
84 p->end =
85 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
86 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
87 }
88 }
89
90 static void
91 w_string(char *s, int n, WFILE *p)
92 {
93 if (p->fp != NULL) {
94 fwrite(s, 1, n, p->fp);
95 }
96 else {
97 while (--n >= 0) {
98 w_byte(*s, p);
99 s++;
100 }
101 }
102 }
103
104 static void
105 w_short(int x, WFILE *p)
106 {
107 w_byte((char)( x & 0xff), p);
108 w_byte((char)((x>> 8) & 0xff), p);
109 }
110
111 static void
112 w_long(long x, WFILE *p)
113 {
114 w_byte((char)( x & 0xff), p);
115 w_byte((char)((x>> 8) & 0xff), p);
116 w_byte((char)((x>>16) & 0xff), p);
117 w_byte((char)((x>>24) & 0xff), p);
118 }
119
120 #if SIZEOF_LONG > 4
121 static void
122 w_long64(long x, WFILE *p)
123 {
124 w_long(x, p);
125 w_long(x>>32, p);
126 }
127 #endif
128
129 /* We assume that Python longs are stored internally in base some power of
130 2**15; for the sake of portability we'll always read and write them in base
131 exactly 2**15. */
132
133 #define PyLong_MARSHAL_SHIFT 15
134 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
135 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
136 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
137 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
138 #endif
139 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
140
141 static void
142 w_PyLong(const PyLongObject *ob, WFILE *p)
143 {
144 Py_ssize_t i, j, n, l;
145 digit d;
146
147 w_byte(TYPE_LONG, p);
148 if (Py_SIZE(ob) == 0) {
149 w_long((long)0, p);
150 return;
151 }
152
153 /* set l to number of base PyLong_MARSHAL_BASE digits */
154 n = ABS(Py_SIZE(ob));
155 l = (n-1) * PyLong_MARSHAL_RATIO;
156 d = ob->ob_digit[n-1];
157 assert(d != 0); /* a PyLong is always normalized */
158 do {
159 d >>= PyLong_MARSHAL_SHIFT;
160 l++;
161 } while (d != 0);
162 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
163
164 for (i=0; i < n-1; i++) {
165 d = ob->ob_digit[i];
166 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
167 w_short(d & PyLong_MARSHAL_MASK, p);
168 d >>= PyLong_MARSHAL_SHIFT;
169 }
170 assert (d == 0);
171 }
172 d = ob->ob_digit[n-1];
173 do {
174 w_short(d & PyLong_MARSHAL_MASK, p);
175 d >>= PyLong_MARSHAL_SHIFT;
176 } while (d != 0);
177 }
178
179 static void
180 w_object(PyObject *v, WFILE *p)
181 {
182 Py_ssize_t i, n;
183
184 p->depth++;
185
186 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
187 p->error = WFERR_NESTEDTOODEEP;
188 }
189 else if (v == NULL) {
190 w_byte(TYPE_NULL, p);
191 }
192 else if (v == Py_None) {
193 w_byte(TYPE_NONE, p);
194 }
195 else if (v == PyExc_StopIteration) {
196 w_byte(TYPE_STOPITER, p);
197 }
198 else if (v == Py_Ellipsis) {
199 w_byte(TYPE_ELLIPSIS, p);
200 }
201 else if (v == Py_False) {
202 w_byte(TYPE_FALSE, p);
203 }
204 else if (v == Py_True) {
205 w_byte(TYPE_TRUE, p);
206 }
207 else if (PyInt_CheckExact(v)) {
208 long x = PyInt_AS_LONG((PyIntObject *)v);
209 #if SIZEOF_LONG > 4
210 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
211 if (y && y != -1) {
212 w_byte(TYPE_INT64, p);
213 w_long64(x, p);
214 }
215 else
216 #endif
217 {
218 w_byte(TYPE_INT, p);
219 w_long(x, p);
220 }
221 }
222 else if (PyLong_CheckExact(v)) {
223 PyLongObject *ob = (PyLongObject *)v;
224 w_PyLong(ob, p);
225 }
226 else if (PyFloat_CheckExact(v)) {
227 if (p->version > 1) {
228 unsigned char buf[8];
229 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
230 buf, 1) < 0) {
231 p->error = WFERR_UNMARSHALLABLE;
232 return;
233 }
234 w_byte(TYPE_BINARY_FLOAT, p);
235 w_string((char*)buf, 8, p);
236 }
237 else {
238 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
239 'g', 17, 0, NULL);
240 if (!buf) {
241 p->error = WFERR_NOMEMORY;
242 return;
243 }
244 n = strlen(buf);
245 w_byte(TYPE_FLOAT, p);
246 w_byte((int)n, p);
247 w_string(buf, (int)n, p);
248 PyMem_Free(buf);
249 }
250 }
251 #ifndef WITHOUT_COMPLEX
252 else if (PyComplex_CheckExact(v)) {
253 if (p->version > 1) {
254 unsigned char buf[8];
255 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
256 buf, 1) < 0) {
257 p->error = WFERR_UNMARSHALLABLE;
258 return;
259 }
260 w_byte(TYPE_BINARY_COMPLEX, p);
261 w_string((char*)buf, 8, p);
262 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
263 buf, 1) < 0) {
264 p->error = WFERR_UNMARSHALLABLE;
265 return;
266 }
267 w_string((char*)buf, 8, p);
268 }
269 else {
270 char *buf;
271 w_byte(TYPE_COMPLEX, p);
272 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
273 'g', 17, 0, NULL);
274 if (!buf) {
275 p->error = WFERR_NOMEMORY;
276 return;
277 }
278 n = strlen(buf);
279 w_byte((int)n, p);
280 w_string(buf, (int)n, p);
281 PyMem_Free(buf);
282 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
283 'g', 17, 0, NULL);
284 if (!buf) {
285 p->error = WFERR_NOMEMORY;
286 return;
287 }
288 n = strlen(buf);
289 w_byte((int)n, p);
290 w_string(buf, (int)n, p);
291 PyMem_Free(buf);
292 }
293 }
294 #endif
295 else if (PyString_CheckExact(v)) {
296 if (p->strings && PyString_CHECK_INTERNED(v)) {
297 PyObject *o = PyDict_GetItem(p->strings, v);
298 if (o) {
299 long w = PyInt_AsLong(o);
300 w_byte(TYPE_STRINGREF, p);
301 w_long(w, p);
302 goto exit;
303 }
304 else {
305 int ok;
306 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
307 ok = o &&
308 PyDict_SetItem(p->strings, v, o) >= 0;
309 Py_XDECREF(o);
310 if (!ok) {
311 p->depth--;
312 p->error = WFERR_UNMARSHALLABLE;
313 return;
314 }
315 w_byte(TYPE_INTERNED, p);
316 }
317 }
318 else {
319 w_byte(TYPE_STRING, p);
320 }
321 n = PyString_GET_SIZE(v);
322 if (n > INT_MAX) {
323 /* huge strings are not supported */
324 p->depth--;
325 p->error = WFERR_UNMARSHALLABLE;
326 return;
327 }
328 w_long((long)n, p);
329 w_string(PyString_AS_STRING(v), (int)n, p);
330 }
331 #ifdef Py_USING_UNICODE
332 else if (PyUnicode_CheckExact(v)) {
333 PyObject *utf8;
334 utf8 = PyUnicode_AsUTF8String(v);
335 if (utf8 == NULL) {
336 p->depth--;
337 p->error = WFERR_UNMARSHALLABLE;
338 return;
339 }
340 w_byte(TYPE_UNICODE, p);
341 n = PyString_GET_SIZE(utf8);
342 if (n > INT_MAX) {
343 p->depth--;
344 p->error = WFERR_UNMARSHALLABLE;
345 return;
346 }
347 w_long((long)n, p);
348 w_string(PyString_AS_STRING(utf8), (int)n, p);
349 Py_DECREF(utf8);
350 }
351 #endif
352 else if (PyTuple_CheckExact(v)) {
353 w_byte(TYPE_TUPLE, p);
354 n = PyTuple_Size(v);
355 w_long((long)n, p);
356 for (i = 0; i < n; i++) {
357 w_object(PyTuple_GET_ITEM(v, i), p);
358 }
359 }
360 else if (PyList_CheckExact(v)) {
361 w_byte(TYPE_LIST, p);
362 n = PyList_GET_SIZE(v);
363 w_long((long)n, p);
364 for (i = 0; i < n; i++) {
365 w_object(PyList_GET_ITEM(v, i), p);
366 }
367 }
368 else if (PyDict_CheckExact(v)) {
369 Py_ssize_t pos;
370 PyObject *key, *value;
371 w_byte(TYPE_DICT, p);
372 /* This one is NULL object terminated! */
373 pos = 0;
374 while (PyDict_Next(v, &pos, &key, &value)) {
375 w_object(key, p);
376 w_object(value, p);
377 }
378 w_object((PyObject *)NULL, p);
379 }
380 else if (PyAnySet_CheckExact(v)) {
381 PyObject *value, *it;
382
383 if (PyObject_TypeCheck(v, &PySet_Type))
384 w_byte(TYPE_SET, p);
385 else
386 w_byte(TYPE_FROZENSET, p);
387 n = PyObject_Size(v);
388 if (n == -1) {
389 p->depth--;
390 p->error = WFERR_UNMARSHALLABLE;
391 return;
392 }
393 w_long((long)n, p);
394 it = PyObject_GetIter(v);
395 if (it == NULL) {
396 p->depth--;
397 p->error = WFERR_UNMARSHALLABLE;
398 return;
399 }
400 while ((value = PyIter_Next(it)) != NULL) {
401 w_object(value, p);
402 Py_DECREF(value);
403 }
404 Py_DECREF(it);
405 if (PyErr_Occurred()) {
406 p->depth--;
407 p->error = WFERR_UNMARSHALLABLE;
408 return;
409 }
410 }
411 else if (PyCode_Check(v)) {
412 PyCodeObject *co = (PyCodeObject *)v;
413 w_byte(TYPE_CODE, p);
414 w_long(co->co_argcount, p);
415 w_long(co->co_nlocals, p);
416 w_long(co->co_stacksize, p);
417 w_long(co->co_flags, p);
418 w_object(co->co_code, p);
419 w_object(co->co_consts, p);
420 w_object(co->co_names, p);
421 w_object(co->co_varnames, p);
422 w_object(co->co_freevars, p);
423 w_object(co->co_cellvars, p);
424 w_object(co->co_filename, p);
425 w_object(co->co_name, p);
426 w_long(co->co_firstlineno, p);
427 w_object(co->co_lnotab, p);
428 }
429 else if (PyObject_CheckReadBuffer(v)) {
430 /* Write unknown buffer-style objects as a string */
431 char *s;
432 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
433 w_byte(TYPE_STRING, p);
434 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
435 if (n > INT_MAX) {
436 p->depth--;
437 p->error = WFERR_UNMARSHALLABLE;
438 return;
439 }
440 w_long((long)n, p);
441 w_string(s, (int)n, p);
442 }
443 else {
444 w_byte(TYPE_UNKNOWN, p);
445 p->error = WFERR_UNMARSHALLABLE;
446 }
447 exit:
448 p->depth--;
449 }
450
451 /* version currently has no effect for writing longs. */
452 void
453 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
454 {
455 WFILE wf;
456 wf.fp = fp;
457 wf.error = WFERR_OK;
458 wf.depth = 0;
459 wf.strings = NULL;
460 wf.version = version;
461 w_long(x, &wf);
462 }
463
464 void
465 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
466 {
467 WFILE wf;
468 wf.fp = fp;
469 wf.error = WFERR_OK;
470 wf.depth = 0;
471 wf.strings = (version > 0) ? PyDict_New() : NULL;
472 wf.version = version;
473 w_object(x, &wf);
474 Py_XDECREF(wf.strings);
475 }
476
477 typedef WFILE RFILE; /* Same struct with different invariants */
478
479 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
480
481 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
482
483 static int
484 r_string(char *s, int n, RFILE *p)
485 {
486 if (p->fp != NULL)
487 /* The result fits into int because it must be <=n. */
488 return (int)fread(s, 1, n, p->fp);
489 if (p->end - p->ptr < n)
490 n = (int)(p->end - p->ptr);
491 memcpy(s, p->ptr, n);
492 p->ptr += n;
493 return n;
494 }
495
496 static int
497 r_short(RFILE *p)
498 {
499 register short x;
500 x = r_byte(p);
501 x |= r_byte(p) << 8;
502 /* Sign-extension, in case short greater than 16 bits */
503 x |= -(x & 0x8000);
504 return x;
505 }
506
507 static long
508 r_long(RFILE *p)
509 {
510 register long x;
511 register FILE *fp = p->fp;
512 if (fp) {
513 x = getc(fp);
514 x |= (long)getc(fp) << 8;
515 x |= (long)getc(fp) << 16;
516 x |= (long)getc(fp) << 24;
517 }
518 else {
519 x = rs_byte(p);
520 x |= (long)rs_byte(p) << 8;
521 x |= (long)rs_byte(p) << 16;
522 x |= (long)rs_byte(p) << 24;
523 }
524 #if SIZEOF_LONG > 4
525 /* Sign extension for 64-bit machines */
526 x |= -(x & 0x80000000L);
527 #endif
528 return x;
529 }
530
531 /* r_long64 deals with the TYPE_INT64 code. On a machine with
532 sizeof(long) > 4, it returns a Python int object, else a Python long
533 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
534 so there's no inefficiency here in returning a PyLong on 32-bit boxes
535 for everything written via TYPE_INT64 (i.e., if an int is written via
536 TYPE_INT64, it *needs* more than 32 bits).
537 */
538 static PyObject *
539 r_long64(RFILE *p)
540 {
541 long lo4 = r_long(p);
542 long hi4 = r_long(p);
543 #if SIZEOF_LONG > 4
544 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
545 return PyInt_FromLong(x);
546 #else
547 unsigned char buf[8];
548 int one = 1;
549 int is_little_endian = (int)*(char*)&one;
550 if (is_little_endian) {
551 memcpy(buf, &lo4, 4);
552 memcpy(buf+4, &hi4, 4);
553 }
554 else {
555 memcpy(buf, &hi4, 4);
556 memcpy(buf+4, &lo4, 4);
557 }
558 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
559 #endif
560 }
561
562 static PyObject *
563 r_PyLong(RFILE *p)
564 {
565 PyLongObject *ob;
566 int size, i, j, md, shorts_in_top_digit;
567 long n;
568 digit d;
569
570 n = r_long(p);
571 if (n == 0)
572 return (PyObject *)_PyLong_New(0);
573 if (n < -INT_MAX || n > INT_MAX) {
574 PyErr_SetString(PyExc_ValueError,
575 "bad marshal data (long size out of range)");
576 return NULL;
577 }
578
579 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
580 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
581 ob = _PyLong_New(size);
582 if (ob == NULL)
583 return NULL;
584 Py_SIZE(ob) = n > 0 ? size : -size;
585
586 for (i = 0; i < size-1; i++) {
587 d = 0;
588 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
589 md = r_short(p);
590 if (md < 0 || md > PyLong_MARSHAL_BASE)
591 goto bad_digit;
592 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
593 }
594 ob->ob_digit[i] = d;
595 }
596 d = 0;
597 for (j=0; j < shorts_in_top_digit; j++) {
598 md = r_short(p);
599 if (md < 0 || md > PyLong_MARSHAL_BASE)
600 goto bad_digit;
601 /* topmost marshal digit should be nonzero */
602 if (md == 0 && j == shorts_in_top_digit - 1) {
603 Py_DECREF(ob);
604 PyErr_SetString(PyExc_ValueError,
605 "bad marshal data (unnormalized long data)");
606 return NULL;
607 }
608 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
609 }
610 /* top digit should be nonzero, else the resulting PyLong won't be
611 normalized */
612 ob->ob_digit[size-1] = d;
613 return (PyObject *)ob;
614 bad_digit:
615 Py_DECREF(ob);
616 PyErr_SetString(PyExc_ValueError,
617 "bad marshal data (digit out of range in long)");
618 return NULL;
619 }
620
621
622 static PyObject *
623 r_object(RFILE *p)
624 {
625 /* NULL is a valid return value, it does not necessarily means that
626 an exception is set. */
627 PyObject *v, *v2;
628 long i, n;
629 int type = r_byte(p);
630 PyObject *retval;
631
632 p->depth++;
633
634 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
635 p->depth--;
636 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
637 return NULL;
638 }
639
640 switch (type) {
641
642 case EOF:
643 PyErr_SetString(PyExc_EOFError,
644 "EOF read where object expected");
645 retval = NULL;
646 break;
647
648 case TYPE_NULL:
649 retval = NULL;
650 break;
651
652 case TYPE_NONE:
653 Py_INCREF(Py_None);
654 retval = Py_None;
655 break;
656
657 case TYPE_STOPITER:
658 Py_INCREF(PyExc_StopIteration);
659 retval = PyExc_StopIteration;
660 break;
661
662 case TYPE_ELLIPSIS:
663 Py_INCREF(Py_Ellipsis);
664 retval = Py_Ellipsis;
665 break;
666
667 case TYPE_FALSE:
668 Py_INCREF(Py_False);
669 retval = Py_False;
670 break;
671
672 case TYPE_TRUE:
673 Py_INCREF(Py_True);
674 retval = Py_True;
675 break;
676
677 case TYPE_INT:
678 retval = PyInt_FromLong(r_long(p));
679 break;
680
681 case TYPE_INT64:
682 retval = r_long64(p);
683 break;
684
685 case TYPE_LONG:
686 retval = r_PyLong(p);
687 break;
688
689 case TYPE_FLOAT:
690 {
691 char buf[256];
692 double dx;
693 n = r_byte(p);
694 if (n == EOF || r_string(buf, (int)n, p) != n) {
695 PyErr_SetString(PyExc_EOFError,
696 "EOF read where object expected");
697 retval = NULL;
698 break;
699 }
700 buf[n] = '\0';
701 dx = PyOS_string_to_double(buf, NULL, NULL);
702 if (dx == -1.0 && PyErr_Occurred())
703 break;
704 retval = PyFloat_FromDouble(dx);
705 break;
706 }
707
708 case TYPE_BINARY_FLOAT:
709 {
710 unsigned char buf[8];
711 double x;
712 if (r_string((char*)buf, 8, p) != 8) {
713 PyErr_SetString(PyExc_EOFError,
714 "EOF read where object expected");
715 retval = NULL;
716 break;
717 }
718 x = _PyFloat_Unpack8(buf, 1);
719 if (x == -1.0 && PyErr_Occurred()) {
720 retval = NULL;
721 break;
722 }
723 retval = PyFloat_FromDouble(x);
724 break;
725 }
726
727 #ifndef WITHOUT_COMPLEX
728 case TYPE_COMPLEX:
729 {
730 char buf[256];
731 Py_complex c;
732 n = r_byte(p);
733 if (n == EOF || r_string(buf, (int)n, p) != n) {
734 PyErr_SetString(PyExc_EOFError,
735 "EOF read where object expected");
736 retval = NULL;
737 break;
738 }
739 buf[n] = '\0';
740 c.real = PyOS_string_to_double(buf, NULL, NULL);
741 if (c.real == -1.0 && PyErr_Occurred())
742 break;
743 n = r_byte(p);
744 if (n == EOF || r_string(buf, (int)n, p) != n) {
745 PyErr_SetString(PyExc_EOFError,
746 "EOF read where object expected");
747 retval = NULL;
748 break;
749 }
750 buf[n] = '\0';
751 c.imag = PyOS_string_to_double(buf, NULL, NULL);
752 if (c.imag == -1.0 && PyErr_Occurred())
753 break;
754 retval = PyComplex_FromCComplex(c);
755 break;
756 }
757
758 case TYPE_BINARY_COMPLEX:
759 {
760 unsigned char buf[8];
761 Py_complex c;
762 if (r_string((char*)buf, 8, p) != 8) {
763 PyErr_SetString(PyExc_EOFError,
764 "EOF read where object expected");
765 retval = NULL;
766 break;
767 }
768 c.real = _PyFloat_Unpack8(buf, 1);
769 if (c.real == -1.0 && PyErr_Occurred()) {
770 retval = NULL;
771 break;
772 }
773 if (r_string((char*)buf, 8, p) != 8) {
774 PyErr_SetString(PyExc_EOFError,
775 "EOF read where object expected");
776 retval = NULL;
777 break;
778 }
779 c.imag = _PyFloat_Unpack8(buf, 1);
780 if (c.imag == -1.0 && PyErr_Occurred()) {
781 retval = NULL;
782 break;
783 }
784 retval = PyComplex_FromCComplex(c);
785 break;
786 }
787 #endif
788
789 case TYPE_INTERNED:
790 case TYPE_STRING:
791 n = r_long(p);
792 if (n < 0 || n > INT_MAX) {
793 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
794 retval = NULL;
795 break;
796 }
797 v = PyString_FromStringAndSize((char *)NULL, n);
798 if (v == NULL) {
799 retval = NULL;
800 break;
801 }
802 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
803 Py_DECREF(v);
804 PyErr_SetString(PyExc_EOFError,
805 "EOF read where object expected");
806 retval = NULL;
807 break;
808 }
809 if (type == TYPE_INTERNED) {
810 PyString_InternInPlace(&v);
811 if (PyList_Append(p->strings, v) < 0) {
812 retval = NULL;
813 break;
814 }
815 }
816 retval = v;
817 break;
818
819 case TYPE_STRINGREF:
820 n = r_long(p);
821 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
822 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)");
823 retval = NULL;
824 break;
825 }
826 v = PyList_GET_ITEM(p->strings, n);
827 Py_INCREF(v);
828 retval = v;
829 break;
830
831 #ifdef Py_USING_UNICODE
832 case TYPE_UNICODE:
833 {
834 char *buffer;
835
836 n = r_long(p);
837 if (n < 0 || n > INT_MAX) {
838 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
839 retval = NULL;
840 break;
841 }
842 buffer = PyMem_NEW(char, n);
843 if (buffer == NULL) {
844 retval = PyErr_NoMemory();
845 break;
846 }
847 if (r_string(buffer, (int)n, p) != n) {
848 PyMem_DEL(buffer);
849 PyErr_SetString(PyExc_EOFError,
850 "EOF read where object expected");
851 retval = NULL;
852 break;
853 }
854 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
855 PyMem_DEL(buffer);
856 retval = v;
857 break;
858 }
859 #endif
860
861 case TYPE_TUPLE:
862 n = r_long(p);
863 if (n < 0 || n > INT_MAX) {
864 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
865 retval = NULL;
866 break;
867 }
868 v = PyTuple_New((int)n);
869 if (v == NULL) {
870 retval = NULL;
871 break;
872 }
873 for (i = 0; i < n; i++) {
874 v2 = r_object(p);
875 if ( v2 == NULL ) {
876 if (!PyErr_Occurred())
877 PyErr_SetString(PyExc_TypeError,
878 "NULL object in marshal data for tuple");
879 Py_DECREF(v);
880 v = NULL;
881 break;
882 }
883 PyTuple_SET_ITEM(v, (int)i, v2);
884 }
885 retval = v;
886 break;
887
888 case TYPE_LIST:
889 n = r_long(p);
890 if (n < 0 || n > INT_MAX) {
891 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
892 retval = NULL;
893 break;
894 }
895 v = PyList_New((int)n);
896 if (v == NULL) {
897 retval = NULL;
898 break;
899 }
900 for (i = 0; i < n; i++) {
901 v2 = r_object(p);
902 if ( v2 == NULL ) {
903 if (!PyErr_Occurred())
904 PyErr_SetString(PyExc_TypeError,
905 "NULL object in marshal data for list");
906 Py_DECREF(v);
907 v = NULL;
908 break;
909 }
910 PyList_SET_ITEM(v, (int)i, v2);
911 }
912 retval = v;
913 break;
914
915 case TYPE_DICT:
916 v = PyDict_New();
917 if (v == NULL) {
918 retval = NULL;
919 break;
920 }
921 for (;;) {
922 PyObject *key, *val;
923 key = r_object(p);
924 if (key == NULL)
925 break;
926 val = r_object(p);
927 if (val != NULL)
928 PyDict_SetItem(v, key, val);
929 Py_DECREF(key);
930 Py_XDECREF(val);
931 }
932 if (PyErr_Occurred()) {
933 Py_DECREF(v);
934 v = NULL;
935 }
936 retval = v;
937 break;
938
939 case TYPE_SET:
940 case TYPE_FROZENSET:
941 n = r_long(p);
942 if (n < 0 || n > INT_MAX) {
943 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
944 retval = NULL;
945 break;
946 }
947 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
948 if (v == NULL) {
949 retval = NULL;
950 break;
951 }
952 for (i = 0; i < n; i++) {
953 v2 = r_object(p);
954 if ( v2 == NULL ) {
955 if (!PyErr_Occurred())
956 PyErr_SetString(PyExc_TypeError,
957 "NULL object in marshal data for set");
958 Py_DECREF(v);
959 v = NULL;
960 break;
961 }
962 if (PySet_Add(v, v2) == -1) {
963 Py_DECREF(v);
964 Py_DECREF(v2);
965 v = NULL;
966 break;
967 }
968 Py_DECREF(v2);
969 }
970 retval = v;
971 break;
972
973 case TYPE_CODE:
974 if (PyEval_GetRestricted()) {
975 PyErr_SetString(PyExc_RuntimeError,
976 "cannot unmarshal code objects in "
977 "restricted execution mode");
978 retval = NULL;
979 break;
980 }
981 else {
982 int argcount;
983 int nlocals;
984 int stacksize;
985 int flags;
986 PyObject *code = NULL;
987 PyObject *consts = NULL;
988 PyObject *names = NULL;
989 PyObject *varnames = NULL;
990 PyObject *freevars = NULL;
991 PyObject *cellvars = NULL;
992 PyObject *filename = NULL;
993 PyObject *name = NULL;
994 int firstlineno;
995 PyObject *lnotab = NULL;
996
997 v = NULL;
998
999 /* XXX ignore long->int overflows for now */
1000 argcount = (int)r_long(p);
1001 nlocals = (int)r_long(p);
1002 stacksize = (int)r_long(p);
1003 flags = (int)r_long(p);
1004 code = r_object(p);
1005 if (code == NULL)
1006 goto code_error;
1007 consts = r_object(p);
1008 if (consts == NULL)
1009 goto code_error;
1010 names = r_object(p);
1011 if (names == NULL)
1012 goto code_error;
1013 varnames = r_object(p);
1014 if (varnames == NULL)
1015 goto code_error;
1016 freevars = r_object(p);
1017 if (freevars == NULL)
1018 goto code_error;
1019 cellvars = r_object(p);
1020 if (cellvars == NULL)
1021 goto code_error;
1022 filename = r_object(p);
1023 if (filename == NULL)
1024 goto code_error;
1025 name = r_object(p);
1026 if (name == NULL)
1027 goto code_error;
1028 firstlineno = (int)r_long(p);
1029 lnotab = r_object(p);
1030 if (lnotab == NULL)
1031 goto code_error;
1032
1033 v = (PyObject *) PyCode_New(
1034 argcount, nlocals, stacksize, flags,
1035 code, consts, names, varnames,
1036 freevars, cellvars, filename, name,
1037 firstlineno, lnotab);
1038
1039 code_error:
1040 Py_XDECREF(code);
1041 Py_XDECREF(consts);
1042 Py_XDECREF(names);
1043 Py_XDECREF(varnames);
1044 Py_XDECREF(freevars);
1045 Py_XDECREF(cellvars);
1046 Py_XDECREF(filename);
1047 Py_XDECREF(name);
1048 Py_XDECREF(lnotab);
1049
1050 }
1051 retval = v;
1052 break;
1053
1054 default:
1055 /* Bogus data got written, which isn't ideal.
1056 This will let you keep working and recover. */
1057 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1058 retval = NULL;
1059 break;
1060
1061 }
1062 p->depth--;
1063 return retval;
1064 }
1065
1066 static PyObject *
1067 read_object(RFILE *p)
1068 {
1069 PyObject *v;
1070 if (PyErr_Occurred()) {
1071 fprintf(stderr, "XXX readobject called with exception set\n");
1072 return NULL;
1073 }
1074 v = r_object(p);
1075 if (v == NULL && !PyErr_Occurred())
1076 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1077 return v;
1078 }
1079
1080 int
1081 PyMarshal_ReadShortFromFile(FILE *fp)
1082 {
1083 RFILE rf;
1084 assert(fp);
1085 rf.fp = fp;
1086 rf.strings = NULL;
1087 rf.end = rf.ptr = NULL;
1088 return r_short(&rf);
1089 }
1090
1091 long
1092 PyMarshal_ReadLongFromFile(FILE *fp)
1093 {
1094 RFILE rf;
1095 rf.fp = fp;
1096 rf.strings = NULL;
1097 rf.ptr = rf.end = NULL;
1098 return r_long(&rf);
1099 }
1100
1101 #ifdef HAVE_FSTAT
1102 /* Return size of file in bytes; < 0 if unknown. */
1103 static off_t
1104 getfilesize(FILE *fp)
1105 {
1106 struct stat st;
1107 if (fstat(fileno(fp), &st) != 0)
1108 return -1;
1109 else
1110 return st.st_size;
1111 }
1112 #endif
1113
1114 /* If we can get the size of the file up-front, and it's reasonably small,
1115 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1116 * than reading a byte at a time from file; speeds .pyc imports.
1117 * CAUTION: since this may read the entire remainder of the file, don't
1118 * call it unless you know you're done with the file.
1119 */
1120 PyObject *
1121 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1122 {
1123 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1124 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1125 */
1126 #define SMALL_FILE_LIMIT (1L << 14)
1127 #define REASONABLE_FILE_LIMIT (1L << 18)
1128 #ifdef HAVE_FSTAT
1129 off_t filesize;
1130 #endif
1131 #ifdef HAVE_FSTAT
1132 filesize = getfilesize(fp);
1133 if (filesize > 0) {
1134 char buf[SMALL_FILE_LIMIT];
1135 char* pBuf = NULL;
1136 if (filesize <= SMALL_FILE_LIMIT)
1137 pBuf = buf;
1138 else if (filesize <= REASONABLE_FILE_LIMIT)
1139 pBuf = (char *)PyMem_MALLOC(filesize);
1140 if (pBuf != NULL) {
1141 PyObject* v;
1142 size_t n;
1143 /* filesize must fit into an int, because it
1144 is smaller than REASONABLE_FILE_LIMIT */
1145 n = fread(pBuf, 1, (int)filesize, fp);
1146 v = PyMarshal_ReadObjectFromString(pBuf, n);
1147 if (pBuf != buf)
1148 PyMem_FREE(pBuf);
1149 return v;
1150 }
1151
1152 }
1153 #endif
1154 /* We don't have fstat, or we do but the file is larger than
1155 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1156 */
1157 return PyMarshal_ReadObjectFromFile(fp);
1158
1159 #undef SMALL_FILE_LIMIT
1160 #undef REASONABLE_FILE_LIMIT
1161 }
1162
1163 PyObject *
1164 PyMarshal_ReadObjectFromFile(FILE *fp)
1165 {
1166 RFILE rf;
1167 PyObject *result;
1168 rf.fp = fp;
1169 rf.strings = PyList_New(0);
1170 rf.depth = 0;
1171 rf.ptr = rf.end = NULL;
1172 result = r_object(&rf);
1173 Py_DECREF(rf.strings);
1174 return result;
1175 }
1176
1177 PyObject *
1178 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1179 {
1180 RFILE rf;
1181 PyObject *result;
1182 rf.fp = NULL;
1183 rf.ptr = str;
1184 rf.end = str + len;
1185 rf.strings = PyList_New(0);
1186 rf.depth = 0;
1187 result = r_object(&rf);
1188 Py_DECREF(rf.strings);
1189 return result;
1190 }
1191
1192 static void
1193 set_error(int error)
1194 {
1195 switch (error) {
1196 case WFERR_NOMEMORY:
1197 PyErr_NoMemory();
1198 break;
1199 case WFERR_UNMARSHALLABLE:
1200 PyErr_SetString(PyExc_ValueError, "unmarshallable object");
1201 break;
1202 case WFERR_NESTEDTOODEEP:
1203 default:
1204 PyErr_SetString(PyExc_ValueError,
1205 "object too deeply nested to marshal");
1206 break;
1207 }
1208 }
1209
1210 PyObject *
1211 PyMarshal_WriteObjectToString(PyObject *x, int version)
1212 {
1213 WFILE wf;
1214 wf.fp = NULL;
1215 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1216 if (wf.str == NULL)
1217 return NULL;
1218 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1219 wf.end = wf.ptr + PyString_Size(wf.str);
1220 wf.error = WFERR_OK;
1221 wf.depth = 0;
1222 wf.version = version;
1223 wf.strings = (version > 0) ? PyDict_New() : NULL;
1224 w_object(x, &wf);
1225 Py_XDECREF(wf.strings);
1226 if (wf.str != NULL) {
1227 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1228 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1229 Py_DECREF(wf.str);
1230 PyErr_SetString(PyExc_OverflowError,
1231 "too much marshall data for a string");
1232 return NULL;
1233 }
1234 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1235 }
1236 if (wf.error != WFERR_OK) {
1237 Py_XDECREF(wf.str);
1238 set_error(wf.error);
1239 return NULL;
1240 }
1241 return wf.str;
1242 }
1243
1244 /* And an interface for Python programs... */
1245
1246 static PyObject *
1247 marshal_dump(PyObject *self, PyObject *args)
1248 {
1249 WFILE wf;
1250 PyObject *x;
1251 PyObject *f;
1252 int version = Py_MARSHAL_VERSION;
1253 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1254 return NULL;
1255 if (!PyFile_Check(f)) {
1256 PyErr_SetString(PyExc_TypeError,
1257 "marshal.dump() 2nd arg must be file");
1258 return NULL;
1259 }
1260 wf.fp = PyFile_AsFile(f);
1261 wf.str = NULL;
1262 wf.ptr = wf.end = NULL;
1263 wf.error = WFERR_OK;
1264 wf.depth = 0;
1265 wf.strings = (version > 0) ? PyDict_New() : 0;
1266 wf.version = version;
1267 w_object(x, &wf);
1268 Py_XDECREF(wf.strings);
1269 if (wf.error != WFERR_OK) {
1270 set_error(wf.error);
1271 return NULL;
1272 }
1273 Py_INCREF(Py_None);
1274 return Py_None;
1275 }
1276
1277 PyDoc_STRVAR(dump_doc,
1278 "dump(value, file[, version])\n\
1279 \n\
1280 Write the value on the open file. The value must be a supported type.\n\
1281 The file must be an open file object such as sys.stdout or returned by\n\
1282 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1283 \n\
1284 If the value has (or contains an object that has) an unsupported type, a\n\
1285 ValueError exception is raised — but garbage data will also be written\n\
1286 to the file. The object will not be properly read back by load()\n\
1287 \n\
1288 New in version 2.4: The version argument indicates the data format that\n\
1289 dump should use.");
1290
1291 static PyObject *
1292 marshal_load(PyObject *self, PyObject *f)
1293 {
1294 RFILE rf;
1295 PyObject *result;
1296 if (!PyFile_Check(f)) {
1297 PyErr_SetString(PyExc_TypeError,
1298 "marshal.load() arg must be file");
1299 return NULL;
1300 }
1301 rf.fp = PyFile_AsFile(f);
1302 rf.strings = PyList_New(0);
1303 rf.depth = 0;
1304 result = read_object(&rf);
1305 Py_DECREF(rf.strings);
1306 return result;
1307 }
1308
1309 PyDoc_STRVAR(load_doc,
1310 "load(file)\n\
1311 \n\
1312 Read one value from the open file and return it. If no valid value is\n\
1313 read (e.g. because the data has a different Python version’s\n\
1314 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1315 The file must be an open file object opened in binary mode ('rb' or\n\
1316 'r+b').\n\
1317 \n\
1318 Note: If an object containing an unsupported type was marshalled with\n\
1319 dump(), load() will substitute None for the unmarshallable type.");
1320
1321
1322 static PyObject *
1323 marshal_dumps(PyObject *self, PyObject *args)
1324 {
1325 PyObject *x;
1326 int version = Py_MARSHAL_VERSION;
1327 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1328 return NULL;
1329 return PyMarshal_WriteObjectToString(x, version);
1330 }
1331
1332 PyDoc_STRVAR(dumps_doc,
1333 "dumps(value[, version])\n\
1334 \n\
1335 Return the string that would be written to a file by dump(value, file).\n\
1336 The value must be a supported type. Raise a ValueError exception if\n\
1337 value has (or contains an object that has) an unsupported type.\n\
1338 \n\
1339 New in version 2.4: The version argument indicates the data format that\n\
1340 dumps should use.");
1341
1342
1343 static PyObject *
1344 marshal_loads(PyObject *self, PyObject *args)
1345 {
1346 RFILE rf;
1347 char *s;
1348 Py_ssize_t n;
1349 PyObject* result;
1350 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1351 return NULL;
1352 rf.fp = NULL;
1353 rf.ptr = s;
1354 rf.end = s + n;
1355 rf.strings = PyList_New(0);
1356 rf.depth = 0;
1357 result = read_object(&rf);
1358 Py_DECREF(rf.strings);
1359 return result;
1360 }
1361
1362 PyDoc_STRVAR(loads_doc,
1363 "loads(string)\n\
1364 \n\
1365 Convert the string to a value. If no valid value is found, raise\n\
1366 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1367 ignored.");
1368
1369 static PyMethodDef marshal_methods[] = {
1370 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1371 {"load", marshal_load, METH_O, load_doc},
1372 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1373 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1374 {NULL, NULL} /* sentinel */
1375 };
1376
1377 PyDoc_STRVAR(marshal_doc,
1378 "This module contains functions that can read and write Python values in\n\
1379 a binary format. The format is specific to Python, but independent of\n\
1380 machine architecture issues.\n\
1381 \n\
1382 Not all Python object types are supported; in general, only objects\n\
1383 whose value is independent from a particular invocation of Python can be\n\
1384 written and read by this module. The following types are supported:\n\
1385 None, integers, long integers, floating point numbers, strings, Unicode\n\
1386 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1387 should be understood that tuples, lists and dictionaries are only\n\
1388 supported as long as the values contained therein are themselves\n\
1389 supported; and recursive lists and dictionaries should not be written\n\
1390 (they will cause infinite loops).\n\
1391 \n\
1392 Variables:\n\
1393 \n\
1394 version -- indicates the format that the module uses. Version 0 is the\n\
1395 historical format, version 1 (added in Python 2.4) shares interned\n\
1396 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1397 floating point numbers. (New in version 2.4)\n\
1398 \n\
1399 Functions:\n\
1400 \n\
1401 dump() -- write value to a file\n\
1402 load() -- read value from a file\n\
1403 dumps() -- write value to a string\n\
1404 loads() -- read value from a string");
1405
1406
1407 PyMODINIT_FUNC
1408 PyMarshal_Init(void)
1409 {
1410 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1411 marshal_doc);
1412 if (mod == NULL)
1413 return;
1414 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1415 }
python git mirror