2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
7 #define PY_SSIZE_T_CLEAN
10 #include "longintrepr.h"
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
20 #define MAX_MARSHAL_STACK_DEPTH 2000
24 #define TYPE_FALSE 'F'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
45 #define TYPE_FROZENSET '>'
51 /* If fp == NULL, the following are valid: */
55 PyObject
*strings
; /* dict on marshal, list on unmarshal */
59 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
60 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
64 w_more(int c
, WFILE
*p
)
66 Py_ssize_t size
, newsize
;
68 return; /* An error already occurred */
69 size
= PyString_Size(p
->str
);
70 newsize
= size
+ size
+ 1024;
71 if (newsize
> 32*1024*1024) {
72 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
74 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
75 p
->ptr
= p
->end
= NULL
;
78 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
80 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
81 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
86 w_string(char *s
, int n
, WFILE
*p
)
89 fwrite(s
, 1, n
, p
->fp
);
100 w_short(int x
, WFILE
*p
)
102 w_byte((char)( x
& 0xff), p
);
103 w_byte((char)((x
>> 8) & 0xff), p
);
107 w_long(long x
, WFILE
*p
)
109 w_byte((char)( x
& 0xff), p
);
110 w_byte((char)((x
>> 8) & 0xff), p
);
111 w_byte((char)((x
>>16) & 0xff), p
);
112 w_byte((char)((x
>>24) & 0xff), p
);
117 w_long64(long x
, WFILE
*p
)
124 /* We assume that Python longs are stored internally in base some power of
125 2**15; for the sake of portability we'll always read and write them in base
128 #define PyLong_MARSHAL_SHIFT 15
129 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
130 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
131 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
132 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
134 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
137 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
139 Py_ssize_t i
, j
, n
, l
;
142 w_byte(TYPE_LONG
, p
);
143 if (Py_SIZE(ob
) == 0) {
148 /* set l to number of base PyLong_MARSHAL_BASE digits */
149 n
= ABS(Py_SIZE(ob
));
150 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
151 d
= ob
->ob_digit
[n
-1];
152 assert(d
!= 0); /* a PyLong is always normalized */
154 d
>>= PyLong_MARSHAL_SHIFT
;
157 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
159 for (i
=0; i
< n
-1; i
++) {
161 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
162 w_short(d
& PyLong_MARSHAL_MASK
, p
);
163 d
>>= PyLong_MARSHAL_SHIFT
;
167 d
= ob
->ob_digit
[n
-1];
169 w_short(d
& PyLong_MARSHAL_MASK
, p
);
170 d
>>= PyLong_MARSHAL_SHIFT
;
175 w_object(PyObject
*v
, WFILE
*p
)
181 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
184 else if (v
== NULL
) {
185 w_byte(TYPE_NULL
, p
);
187 else if (v
== Py_None
) {
188 w_byte(TYPE_NONE
, p
);
190 else if (v
== PyExc_StopIteration
) {
191 w_byte(TYPE_STOPITER
, p
);
193 else if (v
== Py_Ellipsis
) {
194 w_byte(TYPE_ELLIPSIS
, p
);
196 else if (v
== Py_False
) {
197 w_byte(TYPE_FALSE
, p
);
199 else if (v
== Py_True
) {
200 w_byte(TYPE_TRUE
, p
);
202 else if (PyInt_CheckExact(v
)) {
203 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
205 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
207 w_byte(TYPE_INT64
, p
);
217 else if (PyLong_CheckExact(v
)) {
218 PyLongObject
*ob
= (PyLongObject
*)v
;
221 else if (PyFloat_CheckExact(v
)) {
222 if (p
->version
> 1) {
223 unsigned char buf
[8];
224 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
229 w_byte(TYPE_BINARY_FLOAT
, p
);
230 w_string((char*)buf
, 8, p
);
233 char buf
[256]; /* Plenty to format any double */
234 PyFloat_AsReprString(buf
, (PyFloatObject
*)v
);
236 w_byte(TYPE_FLOAT
, p
);
238 w_string(buf
, (int)n
, p
);
241 #ifndef WITHOUT_COMPLEX
242 else if (PyComplex_CheckExact(v
)) {
243 if (p
->version
> 1) {
244 unsigned char buf
[8];
245 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
250 w_byte(TYPE_BINARY_COMPLEX
, p
);
251 w_string((char*)buf
, 8, p
);
252 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
257 w_string((char*)buf
, 8, p
);
260 char buf
[256]; /* Plenty to format any double */
262 w_byte(TYPE_COMPLEX
, p
);
263 temp
= (PyFloatObject
*)PyFloat_FromDouble(
264 PyComplex_RealAsDouble(v
));
269 PyFloat_AsReprString(buf
, temp
);
273 w_string(buf
, (int)n
, p
);
274 temp
= (PyFloatObject
*)PyFloat_FromDouble(
275 PyComplex_ImagAsDouble(v
));
280 PyFloat_AsReprString(buf
, temp
);
284 w_string(buf
, (int)n
, p
);
288 else if (PyString_CheckExact(v
)) {
289 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
290 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
292 long w
= PyInt_AsLong(o
);
293 w_byte(TYPE_STRINGREF
, p
);
299 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
301 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
308 w_byte(TYPE_INTERNED
, p
);
312 w_byte(TYPE_STRING
, p
);
314 n
= PyString_GET_SIZE(v
);
316 /* huge strings are not supported */
322 w_string(PyString_AS_STRING(v
), (int)n
, p
);
324 #ifdef Py_USING_UNICODE
325 else if (PyUnicode_CheckExact(v
)) {
327 utf8
= PyUnicode_AsUTF8String(v
);
333 w_byte(TYPE_UNICODE
, p
);
334 n
= PyString_GET_SIZE(utf8
);
341 w_string(PyString_AS_STRING(utf8
), (int)n
, p
);
345 else if (PyTuple_CheckExact(v
)) {
346 w_byte(TYPE_TUPLE
, p
);
349 for (i
= 0; i
< n
; i
++) {
350 w_object(PyTuple_GET_ITEM(v
, i
), p
);
353 else if (PyList_CheckExact(v
)) {
354 w_byte(TYPE_LIST
, p
);
355 n
= PyList_GET_SIZE(v
);
357 for (i
= 0; i
< n
; i
++) {
358 w_object(PyList_GET_ITEM(v
, i
), p
);
361 else if (PyDict_CheckExact(v
)) {
363 PyObject
*key
, *value
;
364 w_byte(TYPE_DICT
, p
);
365 /* This one is NULL object terminated! */
367 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
371 w_object((PyObject
*)NULL
, p
);
373 else if (PyAnySet_CheckExact(v
)) {
374 PyObject
*value
, *it
;
376 if (PyObject_TypeCheck(v
, &PySet_Type
))
379 w_byte(TYPE_FROZENSET
, p
);
380 n
= PyObject_Size(v
);
387 it
= PyObject_GetIter(v
);
393 while ((value
= PyIter_Next(it
)) != NULL
) {
398 if (PyErr_Occurred()) {
404 else if (PyCode_Check(v
)) {
405 PyCodeObject
*co
= (PyCodeObject
*)v
;
406 w_byte(TYPE_CODE
, p
);
407 w_long(co
->co_argcount
, p
);
408 w_long(co
->co_nlocals
, p
);
409 w_long(co
->co_stacksize
, p
);
410 w_long(co
->co_flags
, p
);
411 w_object(co
->co_code
, p
);
412 w_object(co
->co_consts
, p
);
413 w_object(co
->co_names
, p
);
414 w_object(co
->co_varnames
, p
);
415 w_object(co
->co_freevars
, p
);
416 w_object(co
->co_cellvars
, p
);
417 w_object(co
->co_filename
, p
);
418 w_object(co
->co_name
, p
);
419 w_long(co
->co_firstlineno
, p
);
420 w_object(co
->co_lnotab
, p
);
422 else if (PyObject_CheckReadBuffer(v
)) {
423 /* Write unknown buffer-style objects as a string */
425 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
426 w_byte(TYPE_STRING
, p
);
427 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
434 w_string(s
, (int)n
, p
);
437 w_byte(TYPE_UNKNOWN
, p
);
444 /* version currently has no effect for writing longs. */
446 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
453 wf
.version
= version
;
458 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
464 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
465 wf
.version
= version
;
467 Py_XDECREF(wf
.strings
);
470 typedef WFILE RFILE
; /* Same struct with different invariants */
472 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
474 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
477 r_string(char *s
, int n
, RFILE
*p
)
480 /* The result fits into int because it must be <=n. */
481 return (int)fread(s
, 1, n
, p
->fp
);
482 if (p
->end
- p
->ptr
< n
)
483 n
= (int)(p
->end
- p
->ptr
);
484 memcpy(s
, p
->ptr
, n
);
495 /* Sign-extension, in case short greater than 16 bits */
504 register FILE *fp
= p
->fp
;
507 x
|= (long)getc(fp
) << 8;
508 x
|= (long)getc(fp
) << 16;
509 x
|= (long)getc(fp
) << 24;
513 x
|= (long)rs_byte(p
) << 8;
514 x
|= (long)rs_byte(p
) << 16;
515 x
|= (long)rs_byte(p
) << 24;
518 /* Sign extension for 64-bit machines */
519 x
|= -(x
& 0x80000000L
);
524 /* r_long64 deals with the TYPE_INT64 code. On a machine with
525 sizeof(long) > 4, it returns a Python int object, else a Python long
526 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
527 so there's no inefficiency here in returning a PyLong on 32-bit boxes
528 for everything written via TYPE_INT64 (i.e., if an int is written via
529 TYPE_INT64, it *needs* more than 32 bits).
534 long lo4
= r_long(p
);
535 long hi4
= r_long(p
);
537 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
538 return PyInt_FromLong(x
);
540 unsigned char buf
[8];
542 int is_little_endian
= (int)*(char*)&one
;
543 if (is_little_endian
) {
544 memcpy(buf
, &lo4
, 4);
545 memcpy(buf
+4, &hi4
, 4);
548 memcpy(buf
, &hi4
, 4);
549 memcpy(buf
+4, &lo4
, 4);
551 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
559 int size
, i
, j
, md
, shorts_in_top_digit
;
565 return (PyObject
*)_PyLong_New(0);
566 if (n
< -INT_MAX
|| n
> INT_MAX
) {
567 PyErr_SetString(PyExc_ValueError
,
568 "bad marshal data (long size out of range)");
572 size
= 1 + (ABS(n
) - 1) / PyLong_MARSHAL_RATIO
;
573 shorts_in_top_digit
= 1 + (ABS(n
) - 1) % PyLong_MARSHAL_RATIO
;
574 ob
= _PyLong_New(size
);
577 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
579 for (i
= 0; i
< size
-1; i
++) {
581 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
583 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
585 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
590 for (j
=0; j
< shorts_in_top_digit
; j
++) {
592 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
594 /* topmost marshal digit should be nonzero */
595 if (md
== 0 && j
== shorts_in_top_digit
- 1) {
597 PyErr_SetString(PyExc_ValueError
,
598 "bad marshal data (unnormalized long data)");
601 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
603 /* top digit should be nonzero, else the resulting PyLong won't be
605 ob
->ob_digit
[size
-1] = d
;
606 return (PyObject
*)ob
;
609 PyErr_SetString(PyExc_ValueError
,
610 "bad marshal data (digit out of range in long)");
618 /* NULL is a valid return value, it does not necessarily means that
619 an exception is set. */
622 int type
= r_byte(p
);
627 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
629 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
636 PyErr_SetString(PyExc_EOFError
,
637 "EOF read where object expected");
651 Py_INCREF(PyExc_StopIteration
);
652 retval
= PyExc_StopIteration
;
656 Py_INCREF(Py_Ellipsis
);
657 retval
= Py_Ellipsis
;
671 retval
= PyInt_FromLong(r_long(p
));
675 retval
= r_long64(p
);
679 retval
= r_PyLong(p
);
687 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
688 PyErr_SetString(PyExc_EOFError
,
689 "EOF read where object expected");
695 PyFPE_START_PROTECT("atof", break)
696 dx
= PyOS_ascii_atof(buf
);
697 PyFPE_END_PROTECT(dx
)
698 retval
= PyFloat_FromDouble(dx
);
702 case TYPE_BINARY_FLOAT
:
704 unsigned char buf
[8];
706 if (r_string((char*)buf
, 8, p
) != 8) {
707 PyErr_SetString(PyExc_EOFError
,
708 "EOF read where object expected");
712 x
= _PyFloat_Unpack8(buf
, 1);
713 if (x
== -1.0 && PyErr_Occurred()) {
717 retval
= PyFloat_FromDouble(x
);
721 #ifndef WITHOUT_COMPLEX
727 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
728 PyErr_SetString(PyExc_EOFError
,
729 "EOF read where object expected");
735 PyFPE_START_PROTECT("atof", break;)
736 c
.real
= PyOS_ascii_atof(buf
);
739 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
740 PyErr_SetString(PyExc_EOFError
,
741 "EOF read where object expected");
746 PyFPE_START_PROTECT("atof", break)
747 c
.imag
= PyOS_ascii_atof(buf
);
749 retval
= PyComplex_FromCComplex(c
);
753 case TYPE_BINARY_COMPLEX
:
755 unsigned char buf
[8];
757 if (r_string((char*)buf
, 8, p
) != 8) {
758 PyErr_SetString(PyExc_EOFError
,
759 "EOF read where object expected");
763 c
.real
= _PyFloat_Unpack8(buf
, 1);
764 if (c
.real
== -1.0 && PyErr_Occurred()) {
768 if (r_string((char*)buf
, 8, p
) != 8) {
769 PyErr_SetString(PyExc_EOFError
,
770 "EOF read where object expected");
774 c
.imag
= _PyFloat_Unpack8(buf
, 1);
775 if (c
.imag
== -1.0 && PyErr_Occurred()) {
779 retval
= PyComplex_FromCComplex(c
);
787 if (n
< 0 || n
> INT_MAX
) {
788 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
792 v
= PyString_FromStringAndSize((char *)NULL
, n
);
797 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
799 PyErr_SetString(PyExc_EOFError
,
800 "EOF read where object expected");
804 if (type
== TYPE_INTERNED
) {
805 PyString_InternInPlace(&v
);
806 if (PyList_Append(p
->strings
, v
) < 0) {
816 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
817 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
821 v
= PyList_GET_ITEM(p
->strings
, n
);
826 #ifdef Py_USING_UNICODE
832 if (n
< 0 || n
> INT_MAX
) {
833 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
837 buffer
= PyMem_NEW(char, n
);
838 if (buffer
== NULL
) {
839 retval
= PyErr_NoMemory();
842 if (r_string(buffer
, (int)n
, p
) != n
) {
844 PyErr_SetString(PyExc_EOFError
,
845 "EOF read where object expected");
849 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
858 if (n
< 0 || n
> INT_MAX
) {
859 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
863 v
= PyTuple_New((int)n
);
868 for (i
= 0; i
< n
; i
++) {
871 if (!PyErr_Occurred())
872 PyErr_SetString(PyExc_TypeError
,
873 "NULL object in marshal data for tuple");
878 PyTuple_SET_ITEM(v
, (int)i
, v2
);
885 if (n
< 0 || n
> INT_MAX
) {
886 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
890 v
= PyList_New((int)n
);
895 for (i
= 0; i
< n
; i
++) {
898 if (!PyErr_Occurred())
899 PyErr_SetString(PyExc_TypeError
,
900 "NULL object in marshal data for list");
905 PyList_SET_ITEM(v
, (int)i
, v2
);
923 PyDict_SetItem(v
, key
, val
);
927 if (PyErr_Occurred()) {
937 if (n
< 0 || n
> INT_MAX
) {
938 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
942 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
947 for (i
= 0; i
< n
; i
++) {
950 if (!PyErr_Occurred())
951 PyErr_SetString(PyExc_TypeError
,
952 "NULL object in marshal data for set");
957 if (PySet_Add(v
, v2
) == -1) {
969 if (PyEval_GetRestricted()) {
970 PyErr_SetString(PyExc_RuntimeError
,
971 "cannot unmarshal code objects in "
972 "restricted execution mode");
981 PyObject
*code
= NULL
;
982 PyObject
*consts
= NULL
;
983 PyObject
*names
= NULL
;
984 PyObject
*varnames
= NULL
;
985 PyObject
*freevars
= NULL
;
986 PyObject
*cellvars
= NULL
;
987 PyObject
*filename
= NULL
;
988 PyObject
*name
= NULL
;
990 PyObject
*lnotab
= NULL
;
994 /* XXX ignore long->int overflows for now */
995 argcount
= (int)r_long(p
);
996 nlocals
= (int)r_long(p
);
997 stacksize
= (int)r_long(p
);
998 flags
= (int)r_long(p
);
1002 consts
= r_object(p
);
1005 names
= r_object(p
);
1008 varnames
= r_object(p
);
1009 if (varnames
== NULL
)
1011 freevars
= r_object(p
);
1012 if (freevars
== NULL
)
1014 cellvars
= r_object(p
);
1015 if (cellvars
== NULL
)
1017 filename
= r_object(p
);
1018 if (filename
== NULL
)
1023 firstlineno
= (int)r_long(p
);
1024 lnotab
= r_object(p
);
1028 v
= (PyObject
*) PyCode_New(
1029 argcount
, nlocals
, stacksize
, flags
,
1030 code
, consts
, names
, varnames
,
1031 freevars
, cellvars
, filename
, name
,
1032 firstlineno
, lnotab
);
1038 Py_XDECREF(varnames
);
1039 Py_XDECREF(freevars
);
1040 Py_XDECREF(cellvars
);
1041 Py_XDECREF(filename
);
1050 /* Bogus data got written, which isn't ideal.
1051 This will let you keep working and recover. */
1052 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1062 read_object(RFILE
*p
)
1065 if (PyErr_Occurred()) {
1066 fprintf(stderr
, "XXX readobject called with exception set\n");
1070 if (v
== NULL
&& !PyErr_Occurred())
1071 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1076 PyMarshal_ReadShortFromFile(FILE *fp
)
1082 rf
.end
= rf
.ptr
= NULL
;
1083 return r_short(&rf
);
1087 PyMarshal_ReadLongFromFile(FILE *fp
)
1092 rf
.ptr
= rf
.end
= NULL
;
1097 /* Return size of file in bytes; < 0 if unknown. */
1099 getfilesize(FILE *fp
)
1102 if (fstat(fileno(fp
), &st
) != 0)
1109 /* If we can get the size of the file up-front, and it's reasonably small,
1110 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1111 * than reading a byte at a time from file; speeds .pyc imports.
1112 * CAUTION: since this may read the entire remainder of the file, don't
1113 * call it unless you know you're done with the file.
1116 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1118 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1119 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1121 #define SMALL_FILE_LIMIT (1L << 14)
1122 #define REASONABLE_FILE_LIMIT (1L << 18)
1127 filesize
= getfilesize(fp
);
1129 char buf
[SMALL_FILE_LIMIT
];
1131 if (filesize
<= SMALL_FILE_LIMIT
)
1133 else if (filesize
<= REASONABLE_FILE_LIMIT
)
1134 pBuf
= (char *)PyMem_MALLOC(filesize
);
1138 /* filesize must fit into an int, because it
1139 is smaller than REASONABLE_FILE_LIMIT */
1140 n
= fread(pBuf
, 1, (int)filesize
, fp
);
1141 v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1149 /* We don't have fstat, or we do but the file is larger than
1150 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1152 return PyMarshal_ReadObjectFromFile(fp
);
1154 #undef SMALL_FILE_LIMIT
1155 #undef REASONABLE_FILE_LIMIT
1159 PyMarshal_ReadObjectFromFile(FILE *fp
)
1164 rf
.strings
= PyList_New(0);
1166 rf
.ptr
= rf
.end
= NULL
;
1167 result
= r_object(&rf
);
1168 Py_DECREF(rf
.strings
);
1173 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1180 rf
.strings
= PyList_New(0);
1182 result
= r_object(&rf
);
1183 Py_DECREF(rf
.strings
);
1188 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1192 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1195 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1196 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1199 wf
.version
= version
;
1200 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1202 Py_XDECREF(wf
.strings
);
1203 if (wf
.str
!= NULL
) {
1204 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1205 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1207 PyErr_SetString(PyExc_OverflowError
,
1208 "too much marshall data for a string");
1211 _PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
));
1215 PyErr_SetString(PyExc_ValueError
,
1216 (wf
.error
==1)?"unmarshallable object"
1217 :"object too deeply nested to marshal");
1223 /* And an interface for Python programs... */
1226 marshal_dump(PyObject
*self
, PyObject
*args
)
1231 int version
= Py_MARSHAL_VERSION
;
1232 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1234 if (!PyFile_Check(f
)) {
1235 PyErr_SetString(PyExc_TypeError
,
1236 "marshal.dump() 2nd arg must be file");
1239 wf
.fp
= PyFile_AsFile(f
);
1241 wf
.ptr
= wf
.end
= NULL
;
1244 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1245 wf
.version
= version
;
1247 Py_XDECREF(wf
.strings
);
1249 PyErr_SetString(PyExc_ValueError
,
1250 (wf
.error
==1)?"unmarshallable object"
1251 :"object too deeply nested to marshal");
1258 PyDoc_STRVAR(dump_doc
,
1259 "dump(value, file[, version])\n\
1261 Write the value on the open file. The value must be a supported type.\n\
1262 The file must be an open file object such as sys.stdout or returned by\n\
1263 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1265 If the value has (or contains an object that has) an unsupported type, a\n\
1266 ValueError exception is raised — but garbage data will also be written\n\
1267 to the file. The object will not be properly read back by load()\n\
1269 New in version 2.4: The version argument indicates the data format that\n\
1273 marshal_load(PyObject
*self
, PyObject
*f
)
1277 if (!PyFile_Check(f
)) {
1278 PyErr_SetString(PyExc_TypeError
,
1279 "marshal.load() arg must be file");
1282 rf
.fp
= PyFile_AsFile(f
);
1283 rf
.strings
= PyList_New(0);
1285 result
= read_object(&rf
);
1286 Py_DECREF(rf
.strings
);
1290 PyDoc_STRVAR(load_doc
,
1293 Read one value from the open file and return it. If no valid value is\n\
1294 read (e.g. because the data has a different Python version’s\n\
1295 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1296 The file must be an open file object opened in binary mode ('rb' or\n\
1299 Note: If an object containing an unsupported type was marshalled with\n\
1300 dump(), load() will substitute None for the unmarshallable type.");
1304 marshal_dumps(PyObject
*self
, PyObject
*args
)
1307 int version
= Py_MARSHAL_VERSION
;
1308 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1310 return PyMarshal_WriteObjectToString(x
, version
);
1313 PyDoc_STRVAR(dumps_doc
,
1314 "dumps(value[, version])\n\
1316 Return the string that would be written to a file by dump(value, file).\n\
1317 The value must be a supported type. Raise a ValueError exception if\n\
1318 value has (or contains an object that has) an unsupported type.\n\
1320 New in version 2.4: The version argument indicates the data format that\n\
1321 dumps should use.");
1325 marshal_loads(PyObject
*self
, PyObject
*args
)
1331 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1336 rf
.strings
= PyList_New(0);
1338 result
= read_object(&rf
);
1339 Py_DECREF(rf
.strings
);
1343 PyDoc_STRVAR(loads_doc
,
1346 Convert the string to a value. If no valid value is found, raise\n\
1347 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1350 static PyMethodDef marshal_methods
[] = {
1351 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1352 {"load", marshal_load
, METH_O
, load_doc
},
1353 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1354 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1355 {NULL
, NULL
} /* sentinel */
1358 PyDoc_STRVAR(marshal_doc
,
1359 "This module contains functions that can read and write Python values in\n\
1360 a binary format. The format is specific to Python, but independent of\n\
1361 machine architecture issues.\n\
1363 Not all Python object types are supported; in general, only objects\n\
1364 whose value is independent from a particular invocation of Python can be\n\
1365 written and read by this module. The following types are supported:\n\
1366 None, integers, long integers, floating point numbers, strings, Unicode\n\
1367 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1368 should be understood that tuples, lists and dictionaries are only\n\
1369 supported as long as the values contained therein are themselves\n\
1370 supported; and recursive lists and dictionaries should not be written\n\
1371 (they will cause infinite loops).\n\
1375 version -- indicates the format that the module uses. Version 0 is the\n\
1376 historical format, version 1 (added in Python 2.4) shares interned\n\
1377 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1378 floating point numbers. (New in version 2.4)\n\
1382 dump() -- write value to a file\n\
1383 load() -- read value from a file\n\
1384 dumps() -- write value to a string\n\
1385 loads() -- read value from a string");
1389 PyMarshal_Init(void)
1391 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1395 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);