2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
7 #define PY_SSIZE_T_CLEAN
10 #include "longintrepr.h"
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
20 #define MAX_MARSHAL_STACK_DEPTH 2000
24 #define TYPE_FALSE 'F'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
45 #define TYPE_FROZENSET '>'
51 /* If fp == NULL, the following are valid: */
55 PyObject
*strings
; /* dict on marshal, list on unmarshal */
59 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
60 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
64 w_more(int c
, WFILE
*p
)
66 Py_ssize_t size
, newsize
;
68 return; /* An error already occurred */
69 size
= PyString_Size(p
->str
);
70 newsize
= size
+ size
+ 1024;
71 if (newsize
> 32*1024*1024) {
72 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
74 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
75 p
->ptr
= p
->end
= NULL
;
78 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
80 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
81 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
86 w_string(char *s
, int n
, WFILE
*p
)
89 fwrite(s
, 1, n
, p
->fp
);
100 w_short(int x
, WFILE
*p
)
102 w_byte((char)( x
& 0xff), p
);
103 w_byte((char)((x
>> 8) & 0xff), p
);
107 w_long(long x
, WFILE
*p
)
109 w_byte((char)( x
& 0xff), p
);
110 w_byte((char)((x
>> 8) & 0xff), p
);
111 w_byte((char)((x
>>16) & 0xff), p
);
112 w_byte((char)((x
>>24) & 0xff), p
);
117 w_long64(long x
, WFILE
*p
)
124 /* We assume that Python longs are stored internally in base some power of
125 2**15; for the sake of portability we'll always read and write them in base
128 #define PyLong_MARSHAL_SHIFT 15
129 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
130 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
131 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
132 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
134 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
137 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
139 Py_ssize_t i
, j
, n
, l
;
142 w_byte(TYPE_LONG
, p
);
143 if (Py_SIZE(ob
) == 0) {
148 /* set l to number of base PyLong_MARSHAL_BASE digits */
149 n
= ABS(Py_SIZE(ob
));
150 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
151 d
= ob
->ob_digit
[n
-1];
152 assert(d
!= 0); /* a PyLong is always normalized */
154 d
>>= PyLong_MARSHAL_SHIFT
;
157 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
159 for (i
=0; i
< n
-1; i
++) {
161 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
162 w_short(d
& PyLong_MARSHAL_MASK
, p
);
163 d
>>= PyLong_MARSHAL_SHIFT
;
167 d
= ob
->ob_digit
[n
-1];
169 w_short(d
& PyLong_MARSHAL_MASK
, p
);
170 d
>>= PyLong_MARSHAL_SHIFT
;
175 w_object(PyObject
*v
, WFILE
*p
)
181 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
184 else if (v
== NULL
) {
185 w_byte(TYPE_NULL
, p
);
187 else if (v
== Py_None
) {
188 w_byte(TYPE_NONE
, p
);
190 else if (v
== PyExc_StopIteration
) {
191 w_byte(TYPE_STOPITER
, p
);
193 else if (v
== Py_Ellipsis
) {
194 w_byte(TYPE_ELLIPSIS
, p
);
196 else if (v
== Py_False
) {
197 w_byte(TYPE_FALSE
, p
);
199 else if (v
== Py_True
) {
200 w_byte(TYPE_TRUE
, p
);
202 else if (PyInt_CheckExact(v
)) {
203 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
205 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
207 w_byte(TYPE_INT64
, p
);
217 else if (PyLong_CheckExact(v
)) {
218 PyLongObject
*ob
= (PyLongObject
*)v
;
221 else if (PyFloat_CheckExact(v
)) {
222 if (p
->version
> 1) {
223 unsigned char buf
[8];
224 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
229 w_byte(TYPE_BINARY_FLOAT
, p
);
230 w_string((char*)buf
, 8, p
);
233 char buf
[256]; /* Plenty to format any double */
234 PyFloat_AsReprString(buf
, (PyFloatObject
*)v
);
236 w_byte(TYPE_FLOAT
, p
);
238 w_string(buf
, (int)n
, p
);
241 #ifndef WITHOUT_COMPLEX
242 else if (PyComplex_CheckExact(v
)) {
243 if (p
->version
> 1) {
244 unsigned char buf
[8];
245 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
250 w_byte(TYPE_BINARY_COMPLEX
, p
);
251 w_string((char*)buf
, 8, p
);
252 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
257 w_string((char*)buf
, 8, p
);
260 char buf
[256]; /* Plenty to format any double */
262 w_byte(TYPE_COMPLEX
, p
);
263 temp
= (PyFloatObject
*)PyFloat_FromDouble(
264 PyComplex_RealAsDouble(v
));
269 PyFloat_AsReprString(buf
, temp
);
273 w_string(buf
, (int)n
, p
);
274 temp
= (PyFloatObject
*)PyFloat_FromDouble(
275 PyComplex_ImagAsDouble(v
));
280 PyFloat_AsReprString(buf
, temp
);
284 w_string(buf
, (int)n
, p
);
288 else if (PyString_CheckExact(v
)) {
289 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
290 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
292 long w
= PyInt_AsLong(o
);
293 w_byte(TYPE_STRINGREF
, p
);
299 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
301 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
308 w_byte(TYPE_INTERNED
, p
);
312 w_byte(TYPE_STRING
, p
);
314 n
= PyString_GET_SIZE(v
);
316 /* huge strings are not supported */
322 w_string(PyString_AS_STRING(v
), (int)n
, p
);
324 #ifdef Py_USING_UNICODE
325 else if (PyUnicode_CheckExact(v
)) {
327 utf8
= PyUnicode_AsUTF8String(v
);
333 w_byte(TYPE_UNICODE
, p
);
334 n
= PyString_GET_SIZE(utf8
);
341 w_string(PyString_AS_STRING(utf8
), (int)n
, p
);
345 else if (PyTuple_CheckExact(v
)) {
346 w_byte(TYPE_TUPLE
, p
);
349 for (i
= 0; i
< n
; i
++) {
350 w_object(PyTuple_GET_ITEM(v
, i
), p
);
353 else if (PyList_CheckExact(v
)) {
354 w_byte(TYPE_LIST
, p
);
355 n
= PyList_GET_SIZE(v
);
357 for (i
= 0; i
< n
; i
++) {
358 w_object(PyList_GET_ITEM(v
, i
), p
);
361 else if (PyDict_CheckExact(v
)) {
363 PyObject
*key
, *value
;
364 w_byte(TYPE_DICT
, p
);
365 /* This one is NULL object terminated! */
367 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
371 w_object((PyObject
*)NULL
, p
);
373 else if (PyAnySet_CheckExact(v
)) {
374 PyObject
*value
, *it
;
376 if (PyObject_TypeCheck(v
, &PySet_Type
))
379 w_byte(TYPE_FROZENSET
, p
);
380 n
= PyObject_Size(v
);
387 it
= PyObject_GetIter(v
);
393 while ((value
= PyIter_Next(it
)) != NULL
) {
398 if (PyErr_Occurred()) {
404 else if (PyCode_Check(v
)) {
405 PyCodeObject
*co
= (PyCodeObject
*)v
;
406 w_byte(TYPE_CODE
, p
);
407 w_long(co
->co_argcount
, p
);
408 w_long(co
->co_nlocals
, p
);
409 w_long(co
->co_stacksize
, p
);
410 w_long(co
->co_flags
, p
);
411 w_object(co
->co_code
, p
);
412 w_object(co
->co_consts
, p
);
413 w_object(co
->co_names
, p
);
414 w_object(co
->co_varnames
, p
);
415 w_object(co
->co_freevars
, p
);
416 w_object(co
->co_cellvars
, p
);
417 w_object(co
->co_filename
, p
);
418 w_object(co
->co_name
, p
);
419 w_long(co
->co_firstlineno
, p
);
420 w_object(co
->co_lnotab
, p
);
422 else if (PyObject_CheckReadBuffer(v
)) {
423 /* Write unknown buffer-style objects as a string */
425 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
426 w_byte(TYPE_STRING
, p
);
427 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
434 w_string(s
, (int)n
, p
);
437 w_byte(TYPE_UNKNOWN
, p
);
444 /* version currently has no effect for writing longs. */
446 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
453 wf
.version
= version
;
458 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
464 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
465 wf
.version
= version
;
467 Py_XDECREF(wf
.strings
);
470 typedef WFILE RFILE
; /* Same struct with different invariants */
472 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
474 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
477 r_string(char *s
, int n
, RFILE
*p
)
480 /* The result fits into int because it must be <=n. */
481 return (int)fread(s
, 1, n
, p
->fp
);
482 if (p
->end
- p
->ptr
< n
)
483 n
= (int)(p
->end
- p
->ptr
);
484 memcpy(s
, p
->ptr
, n
);
495 /* Sign-extension, in case short greater than 16 bits */
504 register FILE *fp
= p
->fp
;
507 x
|= (long)getc(fp
) << 8;
508 x
|= (long)getc(fp
) << 16;
509 x
|= (long)getc(fp
) << 24;
513 x
|= (long)rs_byte(p
) << 8;
514 x
|= (long)rs_byte(p
) << 16;
515 x
|= (long)rs_byte(p
) << 24;
518 /* Sign extension for 64-bit machines */
519 x
|= -(x
& 0x80000000L
);
524 /* r_long64 deals with the TYPE_INT64 code. On a machine with
525 sizeof(long) > 4, it returns a Python int object, else a Python long
526 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
527 so there's no inefficiency here in returning a PyLong on 32-bit boxes
528 for everything written via TYPE_INT64 (i.e., if an int is written via
529 TYPE_INT64, it *needs* more than 32 bits).
534 long lo4
= r_long(p
);
535 long hi4
= r_long(p
);
537 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
538 return PyInt_FromLong(x
);
540 unsigned char buf
[8];
542 int is_little_endian
= (int)*(char*)&one
;
543 if (is_little_endian
) {
544 memcpy(buf
, &lo4
, 4);
545 memcpy(buf
+4, &hi4
, 4);
548 memcpy(buf
, &hi4
, 4);
549 memcpy(buf
+4, &lo4
, 4);
551 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
565 return (PyObject
*)_PyLong_New(0);
566 if (n
< -INT_MAX
|| n
> INT_MAX
) {
567 PyErr_SetString(PyExc_ValueError
,
568 "bad marshal data (long size out of range)");
572 size
= 1 + (ABS(n
)-1) / PyLong_MARSHAL_RATIO
;
573 ob
= _PyLong_New(size
);
576 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
578 for (i
= 0; i
< size
-1; i
++) {
580 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
582 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
584 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
589 for (j
=0; j
< (ABS(n
)-1)%PyLong_MARSHAL_RATIO
+ 1; j
++) {
591 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
593 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
595 ob
->ob_digit
[size
-1] = d
;
596 return (PyObject
*)ob
;
599 PyErr_SetString(PyExc_ValueError
,
600 "bad marshal data (digit out of range in long)");
608 /* NULL is a valid return value, it does not necessarily means that
609 an exception is set. */
612 int type
= r_byte(p
);
617 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
619 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
626 PyErr_SetString(PyExc_EOFError
,
627 "EOF read where object expected");
641 Py_INCREF(PyExc_StopIteration
);
642 retval
= PyExc_StopIteration
;
646 Py_INCREF(Py_Ellipsis
);
647 retval
= Py_Ellipsis
;
661 retval
= PyInt_FromLong(r_long(p
));
665 retval
= r_long64(p
);
669 retval
= r_PyLong(p
);
677 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
678 PyErr_SetString(PyExc_EOFError
,
679 "EOF read where object expected");
685 PyFPE_START_PROTECT("atof", break)
686 dx
= PyOS_ascii_atof(buf
);
687 PyFPE_END_PROTECT(dx
)
688 retval
= PyFloat_FromDouble(dx
);
692 case TYPE_BINARY_FLOAT
:
694 unsigned char buf
[8];
696 if (r_string((char*)buf
, 8, p
) != 8) {
697 PyErr_SetString(PyExc_EOFError
,
698 "EOF read where object expected");
702 x
= _PyFloat_Unpack8(buf
, 1);
703 if (x
== -1.0 && PyErr_Occurred()) {
707 retval
= PyFloat_FromDouble(x
);
711 #ifndef WITHOUT_COMPLEX
717 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
718 PyErr_SetString(PyExc_EOFError
,
719 "EOF read where object expected");
725 PyFPE_START_PROTECT("atof", break;)
726 c
.real
= PyOS_ascii_atof(buf
);
729 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
730 PyErr_SetString(PyExc_EOFError
,
731 "EOF read where object expected");
736 PyFPE_START_PROTECT("atof", break)
737 c
.imag
= PyOS_ascii_atof(buf
);
739 retval
= PyComplex_FromCComplex(c
);
743 case TYPE_BINARY_COMPLEX
:
745 unsigned char buf
[8];
747 if (r_string((char*)buf
, 8, p
) != 8) {
748 PyErr_SetString(PyExc_EOFError
,
749 "EOF read where object expected");
753 c
.real
= _PyFloat_Unpack8(buf
, 1);
754 if (c
.real
== -1.0 && PyErr_Occurred()) {
758 if (r_string((char*)buf
, 8, p
) != 8) {
759 PyErr_SetString(PyExc_EOFError
,
760 "EOF read where object expected");
764 c
.imag
= _PyFloat_Unpack8(buf
, 1);
765 if (c
.imag
== -1.0 && PyErr_Occurred()) {
769 retval
= PyComplex_FromCComplex(c
);
777 if (n
< 0 || n
> INT_MAX
) {
778 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
782 v
= PyString_FromStringAndSize((char *)NULL
, n
);
787 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
789 PyErr_SetString(PyExc_EOFError
,
790 "EOF read where object expected");
794 if (type
== TYPE_INTERNED
) {
795 PyString_InternInPlace(&v
);
796 if (PyList_Append(p
->strings
, v
) < 0) {
806 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
807 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
811 v
= PyList_GET_ITEM(p
->strings
, n
);
816 #ifdef Py_USING_UNICODE
822 if (n
< 0 || n
> INT_MAX
) {
823 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
827 buffer
= PyMem_NEW(char, n
);
828 if (buffer
== NULL
) {
829 retval
= PyErr_NoMemory();
832 if (r_string(buffer
, (int)n
, p
) != n
) {
834 PyErr_SetString(PyExc_EOFError
,
835 "EOF read where object expected");
839 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
848 if (n
< 0 || n
> INT_MAX
) {
849 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
853 v
= PyTuple_New((int)n
);
858 for (i
= 0; i
< n
; i
++) {
861 if (!PyErr_Occurred())
862 PyErr_SetString(PyExc_TypeError
,
863 "NULL object in marshal data for tuple");
868 PyTuple_SET_ITEM(v
, (int)i
, v2
);
875 if (n
< 0 || n
> INT_MAX
) {
876 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
880 v
= PyList_New((int)n
);
885 for (i
= 0; i
< n
; i
++) {
888 if (!PyErr_Occurred())
889 PyErr_SetString(PyExc_TypeError
,
890 "NULL object in marshal data for list");
895 PyList_SET_ITEM(v
, (int)i
, v2
);
913 PyDict_SetItem(v
, key
, val
);
917 if (PyErr_Occurred()) {
927 if (n
< 0 || n
> INT_MAX
) {
928 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
932 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
937 for (i
= 0; i
< n
; i
++) {
940 if (!PyErr_Occurred())
941 PyErr_SetString(PyExc_TypeError
,
942 "NULL object in marshal data for set");
947 if (PySet_Add(v
, v2
) == -1) {
959 if (PyEval_GetRestricted()) {
960 PyErr_SetString(PyExc_RuntimeError
,
961 "cannot unmarshal code objects in "
962 "restricted execution mode");
971 PyObject
*code
= NULL
;
972 PyObject
*consts
= NULL
;
973 PyObject
*names
= NULL
;
974 PyObject
*varnames
= NULL
;
975 PyObject
*freevars
= NULL
;
976 PyObject
*cellvars
= NULL
;
977 PyObject
*filename
= NULL
;
978 PyObject
*name
= NULL
;
980 PyObject
*lnotab
= NULL
;
984 /* XXX ignore long->int overflows for now */
985 argcount
= (int)r_long(p
);
986 nlocals
= (int)r_long(p
);
987 stacksize
= (int)r_long(p
);
988 flags
= (int)r_long(p
);
992 consts
= r_object(p
);
998 varnames
= r_object(p
);
999 if (varnames
== NULL
)
1001 freevars
= r_object(p
);
1002 if (freevars
== NULL
)
1004 cellvars
= r_object(p
);
1005 if (cellvars
== NULL
)
1007 filename
= r_object(p
);
1008 if (filename
== NULL
)
1013 firstlineno
= (int)r_long(p
);
1014 lnotab
= r_object(p
);
1018 v
= (PyObject
*) PyCode_New(
1019 argcount
, nlocals
, stacksize
, flags
,
1020 code
, consts
, names
, varnames
,
1021 freevars
, cellvars
, filename
, name
,
1022 firstlineno
, lnotab
);
1028 Py_XDECREF(varnames
);
1029 Py_XDECREF(freevars
);
1030 Py_XDECREF(cellvars
);
1031 Py_XDECREF(filename
);
1040 /* Bogus data got written, which isn't ideal.
1041 This will let you keep working and recover. */
1042 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1052 read_object(RFILE
*p
)
1055 if (PyErr_Occurred()) {
1056 fprintf(stderr
, "XXX readobject called with exception set\n");
1060 if (v
== NULL
&& !PyErr_Occurred())
1061 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1066 PyMarshal_ReadShortFromFile(FILE *fp
)
1072 rf
.end
= rf
.ptr
= NULL
;
1073 return r_short(&rf
);
1077 PyMarshal_ReadLongFromFile(FILE *fp
)
1082 rf
.ptr
= rf
.end
= NULL
;
1087 /* Return size of file in bytes; < 0 if unknown. */
1089 getfilesize(FILE *fp
)
1092 if (fstat(fileno(fp
), &st
) != 0)
1099 /* If we can get the size of the file up-front, and it's reasonably small,
1100 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1101 * than reading a byte at a time from file; speeds .pyc imports.
1102 * CAUTION: since this may read the entire remainder of the file, don't
1103 * call it unless you know you're done with the file.
1106 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1108 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1109 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1111 #define SMALL_FILE_LIMIT (1L << 14)
1112 #define REASONABLE_FILE_LIMIT (1L << 18)
1117 filesize
= getfilesize(fp
);
1119 char buf
[SMALL_FILE_LIMIT
];
1121 if (filesize
<= SMALL_FILE_LIMIT
)
1123 else if (filesize
<= REASONABLE_FILE_LIMIT
)
1124 pBuf
= (char *)PyMem_MALLOC(filesize
);
1128 /* filesize must fit into an int, because it
1129 is smaller than REASONABLE_FILE_LIMIT */
1130 n
= fread(pBuf
, 1, (int)filesize
, fp
);
1131 v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1139 /* We don't have fstat, or we do but the file is larger than
1140 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1142 return PyMarshal_ReadObjectFromFile(fp
);
1144 #undef SMALL_FILE_LIMIT
1145 #undef REASONABLE_FILE_LIMIT
1149 PyMarshal_ReadObjectFromFile(FILE *fp
)
1154 rf
.strings
= PyList_New(0);
1156 rf
.ptr
= rf
.end
= NULL
;
1157 result
= r_object(&rf
);
1158 Py_DECREF(rf
.strings
);
1163 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1170 rf
.strings
= PyList_New(0);
1172 result
= r_object(&rf
);
1173 Py_DECREF(rf
.strings
);
1178 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1182 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1185 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1186 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1189 wf
.version
= version
;
1190 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1192 Py_XDECREF(wf
.strings
);
1193 if (wf
.str
!= NULL
) {
1194 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1195 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1197 PyErr_SetString(PyExc_OverflowError
,
1198 "too much marshall data for a string");
1201 _PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
));
1205 PyErr_SetString(PyExc_ValueError
,
1206 (wf
.error
==1)?"unmarshallable object"
1207 :"object too deeply nested to marshal");
1213 /* And an interface for Python programs... */
1216 marshal_dump(PyObject
*self
, PyObject
*args
)
1221 int version
= Py_MARSHAL_VERSION
;
1222 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1224 if (!PyFile_Check(f
)) {
1225 PyErr_SetString(PyExc_TypeError
,
1226 "marshal.dump() 2nd arg must be file");
1229 wf
.fp
= PyFile_AsFile(f
);
1231 wf
.ptr
= wf
.end
= NULL
;
1234 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1235 wf
.version
= version
;
1237 Py_XDECREF(wf
.strings
);
1239 PyErr_SetString(PyExc_ValueError
,
1240 (wf
.error
==1)?"unmarshallable object"
1241 :"object too deeply nested to marshal");
1248 PyDoc_STRVAR(dump_doc
,
1249 "dump(value, file[, version])\n\
1251 Write the value on the open file. The value must be a supported type.\n\
1252 The file must be an open file object such as sys.stdout or returned by\n\
1253 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1255 If the value has (or contains an object that has) an unsupported type, a\n\
1256 ValueError exception is raised — but garbage data will also be written\n\
1257 to the file. The object will not be properly read back by load()\n\
1259 New in version 2.4: The version argument indicates the data format that\n\
1263 marshal_load(PyObject
*self
, PyObject
*f
)
1267 if (!PyFile_Check(f
)) {
1268 PyErr_SetString(PyExc_TypeError
,
1269 "marshal.load() arg must be file");
1272 rf
.fp
= PyFile_AsFile(f
);
1273 rf
.strings
= PyList_New(0);
1275 result
= read_object(&rf
);
1276 Py_DECREF(rf
.strings
);
1280 PyDoc_STRVAR(load_doc
,
1283 Read one value from the open file and return it. If no valid value is\n\
1284 read (e.g. because the data has a different Python version’s\n\
1285 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1286 The file must be an open file object opened in binary mode ('rb' or\n\
1289 Note: If an object containing an unsupported type was marshalled with\n\
1290 dump(), load() will substitute None for the unmarshallable type.");
1294 marshal_dumps(PyObject
*self
, PyObject
*args
)
1297 int version
= Py_MARSHAL_VERSION
;
1298 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1300 return PyMarshal_WriteObjectToString(x
, version
);
1303 PyDoc_STRVAR(dumps_doc
,
1304 "dumps(value[, version])\n\
1306 Return the string that would be written to a file by dump(value, file).\n\
1307 The value must be a supported type. Raise a ValueError exception if\n\
1308 value has (or contains an object that has) an unsupported type.\n\
1310 New in version 2.4: The version argument indicates the data format that\n\
1311 dumps should use.");
1315 marshal_loads(PyObject
*self
, PyObject
*args
)
1321 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1326 rf
.strings
= PyList_New(0);
1328 result
= read_object(&rf
);
1329 Py_DECREF(rf
.strings
);
1333 PyDoc_STRVAR(loads_doc
,
1336 Convert the string to a value. If no valid value is found, raise\n\
1337 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1340 static PyMethodDef marshal_methods
[] = {
1341 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1342 {"load", marshal_load
, METH_O
, load_doc
},
1343 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1344 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1345 {NULL
, NULL
} /* sentinel */
1348 PyDoc_STRVAR(marshal_doc
,
1349 "This module contains functions that can read and write Python values in\n\
1350 a binary format. The format is specific to Python, but independent of\n\
1351 machine architecture issues.\n\
1353 Not all Python object types are supported; in general, only objects\n\
1354 whose value is independent from a particular invocation of Python can be\n\
1355 written and read by this module. The following types are supported:\n\
1356 None, integers, long integers, floating point numbers, strings, Unicode\n\
1357 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1358 should be understood that tuples, lists and dictionaries are only\n\
1359 supported as long as the values contained therein are themselves\n\
1360 supported; and recursive lists and dictionaries should not be written\n\
1361 (they will cause infinite loops).\n\
1365 version -- indicates the format that the module uses. Version 0 is the\n\
1366 historical format, version 1 (added in Python 2.4) shares interned\n\
1367 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1368 floating point numbers. (New in version 2.4)\n\
1372 dump() -- write value to a file\n\
1373 load() -- read value from a file\n\
1374 dumps() -- write value to a string\n\
1375 loads() -- read value from a string");
1379 PyMarshal_Init(void)
1381 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1385 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);