Issue #7051: Clarify behaviour of 'g' and 'G'-style formatting.
[python.git] / Modules / _json.c
blobcfe87087d7f3c6bd78b98213516fb4430ae9556c
1 #include "Python.h"
2 #include "structmember.h"
3 #if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE)
4 #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
5 #endif
6 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
7 typedef int Py_ssize_t;
8 #define PY_SSIZE_T_MAX INT_MAX
9 #define PY_SSIZE_T_MIN INT_MIN
10 #define PyInt_FromSsize_t PyInt_FromLong
11 #define PyInt_AsSsize_t PyInt_AsLong
12 #endif
13 #ifndef Py_IS_FINITE
14 #define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X))
15 #endif
17 #ifdef __GNUC__
18 #define UNUSED __attribute__((__unused__))
19 #else
20 #define UNUSED
21 #endif
23 #define DEFAULT_ENCODING "utf-8"
25 #define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType)
26 #define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType)
27 #define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType)
28 #define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType)
30 static PyTypeObject PyScannerType;
31 static PyTypeObject PyEncoderType;
33 typedef struct _PyScannerObject {
34 PyObject_HEAD
35 PyObject *encoding;
36 PyObject *strict;
37 PyObject *object_hook;
38 PyObject *pairs_hook;
39 PyObject *parse_float;
40 PyObject *parse_int;
41 PyObject *parse_constant;
42 } PyScannerObject;
44 static PyMemberDef scanner_members[] = {
45 {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"},
46 {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"},
47 {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"},
48 {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"},
49 {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"},
50 {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"},
51 {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"},
52 {NULL}
55 typedef struct _PyEncoderObject {
56 PyObject_HEAD
57 PyObject *markers;
58 PyObject *defaultfn;
59 PyObject *encoder;
60 PyObject *indent;
61 PyObject *key_separator;
62 PyObject *item_separator;
63 PyObject *sort_keys;
64 PyObject *skipkeys;
65 int fast_encode;
66 int allow_nan;
67 } PyEncoderObject;
69 static PyMemberDef encoder_members[] = {
70 {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"},
71 {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"},
72 {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"},
73 {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"},
74 {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"},
75 {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"},
76 {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"},
77 {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"},
78 {NULL}
81 static Py_ssize_t
82 ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars);
83 static PyObject *
84 ascii_escape_unicode(PyObject *pystr);
85 static PyObject *
86 ascii_escape_str(PyObject *pystr);
87 static PyObject *
88 py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr);
89 void init_json(void);
90 static PyObject *
91 scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
92 static PyObject *
93 scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
94 static PyObject *
95 _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx);
96 static PyObject *
97 scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
98 static int
99 scanner_init(PyObject *self, PyObject *args, PyObject *kwds);
100 static void
101 scanner_dealloc(PyObject *self);
102 static int
103 scanner_clear(PyObject *self);
104 static PyObject *
105 encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
106 static int
107 encoder_init(PyObject *self, PyObject *args, PyObject *kwds);
108 static void
109 encoder_dealloc(PyObject *self);
110 static int
111 encoder_clear(PyObject *self);
112 static int
113 encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level);
114 static int
115 encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level);
116 static int
117 encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level);
118 static PyObject *
119 _encoded_const(PyObject *obj);
120 static void
121 raise_errmsg(char *msg, PyObject *s, Py_ssize_t end);
122 static PyObject *
123 encoder_encode_string(PyEncoderObject *s, PyObject *obj);
124 static int
125 _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr);
126 static PyObject *
127 _convertPyInt_FromSsize_t(Py_ssize_t *size_ptr);
128 static PyObject *
129 encoder_encode_float(PyEncoderObject *s, PyObject *obj);
131 #define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"')
132 #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r'))
134 #define MIN_EXPANSION 6
135 #ifdef Py_UNICODE_WIDE
136 #define MAX_EXPANSION (2 * MIN_EXPANSION)
137 #else
138 #define MAX_EXPANSION MIN_EXPANSION
139 #endif
141 static int
142 _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr)
144 /* PyObject to Py_ssize_t converter */
145 *size_ptr = PyInt_AsSsize_t(o);
146 if (*size_ptr == -1 && PyErr_Occurred())
147 return 0;
148 return 1;
151 static PyObject *
152 _convertPyInt_FromSsize_t(Py_ssize_t *size_ptr)
154 /* Py_ssize_t to PyObject converter */
155 return PyInt_FromSsize_t(*size_ptr);
158 static Py_ssize_t
159 ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars)
161 /* Escape unicode code point c to ASCII escape sequences
162 in char *output. output must have at least 12 bytes unused to
163 accommodate an escaped surrogate pair "\uXXXX\uXXXX" */
164 output[chars++] = '\\';
165 switch (c) {
166 case '\\': output[chars++] = (char)c; break;
167 case '"': output[chars++] = (char)c; break;
168 case '\b': output[chars++] = 'b'; break;
169 case '\f': output[chars++] = 'f'; break;
170 case '\n': output[chars++] = 'n'; break;
171 case '\r': output[chars++] = 'r'; break;
172 case '\t': output[chars++] = 't'; break;
173 default:
174 #ifdef Py_UNICODE_WIDE
175 if (c >= 0x10000) {
176 /* UTF-16 surrogate pair */
177 Py_UNICODE v = c - 0x10000;
178 c = 0xd800 | ((v >> 10) & 0x3ff);
179 output[chars++] = 'u';
180 output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf];
181 output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf];
182 output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf];
183 output[chars++] = "0123456789abcdef"[(c ) & 0xf];
184 c = 0xdc00 | (v & 0x3ff);
185 output[chars++] = '\\';
187 #endif
188 output[chars++] = 'u';
189 output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf];
190 output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf];
191 output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf];
192 output[chars++] = "0123456789abcdef"[(c ) & 0xf];
194 return chars;
197 static PyObject *
198 ascii_escape_unicode(PyObject *pystr)
200 /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */
201 Py_ssize_t i;
202 Py_ssize_t input_chars;
203 Py_ssize_t output_size;
204 Py_ssize_t max_output_size;
205 Py_ssize_t chars;
206 PyObject *rval;
207 char *output;
208 Py_UNICODE *input_unicode;
210 input_chars = PyUnicode_GET_SIZE(pystr);
211 input_unicode = PyUnicode_AS_UNICODE(pystr);
213 /* One char input can be up to 6 chars output, estimate 4 of these */
214 output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
215 max_output_size = 2 + (input_chars * MAX_EXPANSION);
216 rval = PyString_FromStringAndSize(NULL, output_size);
217 if (rval == NULL) {
218 return NULL;
220 output = PyString_AS_STRING(rval);
221 chars = 0;
222 output[chars++] = '"';
223 for (i = 0; i < input_chars; i++) {
224 Py_UNICODE c = input_unicode[i];
225 if (S_CHAR(c)) {
226 output[chars++] = (char)c;
228 else {
229 chars = ascii_escape_char(c, output, chars);
231 if (output_size - chars < (1 + MAX_EXPANSION)) {
232 /* There's more than four, so let's resize by a lot */
233 Py_ssize_t new_output_size = output_size * 2;
234 /* This is an upper bound */
235 if (new_output_size > max_output_size) {
236 new_output_size = max_output_size;
238 /* Make sure that the output size changed before resizing */
239 if (new_output_size != output_size) {
240 output_size = new_output_size;
241 if (_PyString_Resize(&rval, output_size) == -1) {
242 return NULL;
244 output = PyString_AS_STRING(rval);
248 output[chars++] = '"';
249 if (_PyString_Resize(&rval, chars) == -1) {
250 return NULL;
252 return rval;
255 static PyObject *
256 ascii_escape_str(PyObject *pystr)
258 /* Take a PyString pystr and return a new ASCII-only escaped PyString */
259 Py_ssize_t i;
260 Py_ssize_t input_chars;
261 Py_ssize_t output_size;
262 Py_ssize_t chars;
263 PyObject *rval;
264 char *output;
265 char *input_str;
267 input_chars = PyString_GET_SIZE(pystr);
268 input_str = PyString_AS_STRING(pystr);
270 /* Fast path for a string that's already ASCII */
271 for (i = 0; i < input_chars; i++) {
272 Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i];
273 if (!S_CHAR(c)) {
274 /* If we have to escape something, scan the string for unicode */
275 Py_ssize_t j;
276 for (j = i; j < input_chars; j++) {
277 c = (Py_UNICODE)(unsigned char)input_str[j];
278 if (c > 0x7f) {
279 /* We hit a non-ASCII character, bail to unicode mode */
280 PyObject *uni;
281 uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict");
282 if (uni == NULL) {
283 return NULL;
285 rval = ascii_escape_unicode(uni);
286 Py_DECREF(uni);
287 return rval;
290 break;
294 if (i == input_chars) {
295 /* Input is already ASCII */
296 output_size = 2 + input_chars;
298 else {
299 /* One char input can be up to 6 chars output, estimate 4 of these */
300 output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
302 rval = PyString_FromStringAndSize(NULL, output_size);
303 if (rval == NULL) {
304 return NULL;
306 output = PyString_AS_STRING(rval);
307 output[0] = '"';
309 /* We know that everything up to i is ASCII already */
310 chars = i + 1;
311 memcpy(&output[1], input_str, i);
313 for (; i < input_chars; i++) {
314 Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i];
315 if (S_CHAR(c)) {
316 output[chars++] = (char)c;
318 else {
319 chars = ascii_escape_char(c, output, chars);
321 /* An ASCII char can't possibly expand to a surrogate! */
322 if (output_size - chars < (1 + MIN_EXPANSION)) {
323 /* There's more than four, so let's resize by a lot */
324 output_size *= 2;
325 if (output_size > 2 + (input_chars * MIN_EXPANSION)) {
326 output_size = 2 + (input_chars * MIN_EXPANSION);
328 if (_PyString_Resize(&rval, output_size) == -1) {
329 return NULL;
331 output = PyString_AS_STRING(rval);
334 output[chars++] = '"';
335 if (_PyString_Resize(&rval, chars) == -1) {
336 return NULL;
338 return rval;
341 static void
342 raise_errmsg(char *msg, PyObject *s, Py_ssize_t end)
344 /* Use the Python function json.decoder.errmsg to raise a nice
345 looking ValueError exception */
346 static PyObject *errmsg_fn = NULL;
347 PyObject *pymsg;
348 if (errmsg_fn == NULL) {
349 PyObject *decoder = PyImport_ImportModule("json.decoder");
350 if (decoder == NULL)
351 return;
352 errmsg_fn = PyObject_GetAttrString(decoder, "errmsg");
353 Py_DECREF(decoder);
354 if (errmsg_fn == NULL)
355 return;
357 pymsg = PyObject_CallFunction(errmsg_fn, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end);
358 if (pymsg) {
359 PyErr_SetObject(PyExc_ValueError, pymsg);
360 Py_DECREF(pymsg);
364 static PyObject *
365 join_list_unicode(PyObject *lst)
367 /* return u''.join(lst) */
368 static PyObject *joinfn = NULL;
369 if (joinfn == NULL) {
370 PyObject *ustr = PyUnicode_FromUnicode(NULL, 0);
371 if (ustr == NULL)
372 return NULL;
374 joinfn = PyObject_GetAttrString(ustr, "join");
375 Py_DECREF(ustr);
376 if (joinfn == NULL)
377 return NULL;
379 return PyObject_CallFunctionObjArgs(joinfn, lst, NULL);
382 static PyObject *
383 join_list_string(PyObject *lst)
385 /* return ''.join(lst) */
386 static PyObject *joinfn = NULL;
387 if (joinfn == NULL) {
388 PyObject *ustr = PyString_FromStringAndSize(NULL, 0);
389 if (ustr == NULL)
390 return NULL;
392 joinfn = PyObject_GetAttrString(ustr, "join");
393 Py_DECREF(ustr);
394 if (joinfn == NULL)
395 return NULL;
397 return PyObject_CallFunctionObjArgs(joinfn, lst, NULL);
400 static PyObject *
401 _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) {
402 /* return (rval, idx) tuple, stealing reference to rval */
403 PyObject *tpl;
404 PyObject *pyidx;
406 steal a reference to rval, returns (rval, idx)
408 if (rval == NULL) {
409 return NULL;
411 pyidx = PyInt_FromSsize_t(idx);
412 if (pyidx == NULL) {
413 Py_DECREF(rval);
414 return NULL;
416 tpl = PyTuple_New(2);
417 if (tpl == NULL) {
418 Py_DECREF(pyidx);
419 Py_DECREF(rval);
420 return NULL;
422 PyTuple_SET_ITEM(tpl, 0, rval);
423 PyTuple_SET_ITEM(tpl, 1, pyidx);
424 return tpl;
427 static PyObject *
428 scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr)
430 /* Read the JSON string from PyString pystr.
431 end is the index of the first character after the quote.
432 encoding is the encoding of pystr (must be an ASCII superset)
433 if strict is zero then literal control characters are allowed
434 *next_end_ptr is a return-by-reference index of the character
435 after the end quote
437 Return value is a new PyString (if ASCII-only) or PyUnicode
439 PyObject *rval;
440 Py_ssize_t len = PyString_GET_SIZE(pystr);
441 Py_ssize_t begin = end - 1;
442 Py_ssize_t next = begin;
443 int has_unicode = 0;
444 char *buf = PyString_AS_STRING(pystr);
445 PyObject *chunks = PyList_New(0);
446 if (chunks == NULL) {
447 goto bail;
449 if (end < 0 || len <= end) {
450 PyErr_SetString(PyExc_ValueError, "end is out of bounds");
451 goto bail;
453 while (1) {
454 /* Find the end of the string or the next escape */
455 Py_UNICODE c = 0;
456 PyObject *chunk = NULL;
457 for (next = end; next < len; next++) {
458 c = (unsigned char)buf[next];
459 if (c == '"' || c == '\\') {
460 break;
462 else if (strict && c <= 0x1f) {
463 raise_errmsg("Invalid control character at", pystr, next);
464 goto bail;
466 else if (c > 0x7f) {
467 has_unicode = 1;
470 if (!(c == '"' || c == '\\')) {
471 raise_errmsg("Unterminated string starting at", pystr, begin);
472 goto bail;
474 /* Pick up this chunk if it's not zero length */
475 if (next != end) {
476 PyObject *strchunk = PyString_FromStringAndSize(&buf[end], next - end);
477 if (strchunk == NULL) {
478 goto bail;
480 if (has_unicode) {
481 chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL);
482 Py_DECREF(strchunk);
483 if (chunk == NULL) {
484 goto bail;
487 else {
488 chunk = strchunk;
490 if (PyList_Append(chunks, chunk)) {
491 Py_DECREF(chunk);
492 goto bail;
494 Py_DECREF(chunk);
496 next++;
497 if (c == '"') {
498 end = next;
499 break;
501 if (next == len) {
502 raise_errmsg("Unterminated string starting at", pystr, begin);
503 goto bail;
505 c = buf[next];
506 if (c != 'u') {
507 /* Non-unicode backslash escapes */
508 end = next + 1;
509 switch (c) {
510 case '"': break;
511 case '\\': break;
512 case '/': break;
513 case 'b': c = '\b'; break;
514 case 'f': c = '\f'; break;
515 case 'n': c = '\n'; break;
516 case 'r': c = '\r'; break;
517 case 't': c = '\t'; break;
518 default: c = 0;
520 if (c == 0) {
521 raise_errmsg("Invalid \\escape", pystr, end - 2);
522 goto bail;
525 else {
526 c = 0;
527 next++;
528 end = next + 4;
529 if (end >= len) {
530 raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
531 goto bail;
533 /* Decode 4 hex digits */
534 for (; next < end; next++) {
535 Py_UNICODE digit = buf[next];
536 c <<= 4;
537 switch (digit) {
538 case '0': case '1': case '2': case '3': case '4':
539 case '5': case '6': case '7': case '8': case '9':
540 c |= (digit - '0'); break;
541 case 'a': case 'b': case 'c': case 'd': case 'e':
542 case 'f':
543 c |= (digit - 'a' + 10); break;
544 case 'A': case 'B': case 'C': case 'D': case 'E':
545 case 'F':
546 c |= (digit - 'A' + 10); break;
547 default:
548 raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
549 goto bail;
552 #ifdef Py_UNICODE_WIDE
553 /* Surrogate pair */
554 if ((c & 0xfc00) == 0xd800) {
555 Py_UNICODE c2 = 0;
556 if (end + 6 >= len) {
557 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
558 goto bail;
560 if (buf[next++] != '\\' || buf[next++] != 'u') {
561 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
562 goto bail;
564 end += 6;
565 /* Decode 4 hex digits */
566 for (; next < end; next++) {
567 c2 <<= 4;
568 Py_UNICODE digit = buf[next];
569 switch (digit) {
570 case '0': case '1': case '2': case '3': case '4':
571 case '5': case '6': case '7': case '8': case '9':
572 c2 |= (digit - '0'); break;
573 case 'a': case 'b': case 'c': case 'd': case 'e':
574 case 'f':
575 c2 |= (digit - 'a' + 10); break;
576 case 'A': case 'B': case 'C': case 'D': case 'E':
577 case 'F':
578 c2 |= (digit - 'A' + 10); break;
579 default:
580 raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
581 goto bail;
584 if ((c2 & 0xfc00) != 0xdc00) {
585 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
586 goto bail;
588 c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
590 else if ((c & 0xfc00) == 0xdc00) {
591 raise_errmsg("Unpaired low surrogate", pystr, end - 5);
592 goto bail;
594 #endif
596 if (c > 0x7f) {
597 has_unicode = 1;
599 if (has_unicode) {
600 chunk = PyUnicode_FromUnicode(&c, 1);
601 if (chunk == NULL) {
602 goto bail;
605 else {
606 char c_char = Py_CHARMASK(c);
607 chunk = PyString_FromStringAndSize(&c_char, 1);
608 if (chunk == NULL) {
609 goto bail;
612 if (PyList_Append(chunks, chunk)) {
613 Py_DECREF(chunk);
614 goto bail;
616 Py_DECREF(chunk);
619 rval = join_list_string(chunks);
620 if (rval == NULL) {
621 goto bail;
623 Py_CLEAR(chunks);
624 *next_end_ptr = end;
625 return rval;
626 bail:
627 *next_end_ptr = -1;
628 Py_XDECREF(chunks);
629 return NULL;
633 static PyObject *
634 scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr)
636 /* Read the JSON string from PyUnicode pystr.
637 end is the index of the first character after the quote.
638 if strict is zero then literal control characters are allowed
639 *next_end_ptr is a return-by-reference index of the character
640 after the end quote
642 Return value is a new PyUnicode
644 PyObject *rval;
645 Py_ssize_t len = PyUnicode_GET_SIZE(pystr);
646 Py_ssize_t begin = end - 1;
647 Py_ssize_t next = begin;
648 const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr);
649 PyObject *chunks = PyList_New(0);
650 if (chunks == NULL) {
651 goto bail;
653 if (end < 0 || len <= end) {
654 PyErr_SetString(PyExc_ValueError, "end is out of bounds");
655 goto bail;
657 while (1) {
658 /* Find the end of the string or the next escape */
659 Py_UNICODE c = 0;
660 PyObject *chunk = NULL;
661 for (next = end; next < len; next++) {
662 c = buf[next];
663 if (c == '"' || c == '\\') {
664 break;
666 else if (strict && c <= 0x1f) {
667 raise_errmsg("Invalid control character at", pystr, next);
668 goto bail;
671 if (!(c == '"' || c == '\\')) {
672 raise_errmsg("Unterminated string starting at", pystr, begin);
673 goto bail;
675 /* Pick up this chunk if it's not zero length */
676 if (next != end) {
677 chunk = PyUnicode_FromUnicode(&buf[end], next - end);
678 if (chunk == NULL) {
679 goto bail;
681 if (PyList_Append(chunks, chunk)) {
682 Py_DECREF(chunk);
683 goto bail;
685 Py_DECREF(chunk);
687 next++;
688 if (c == '"') {
689 end = next;
690 break;
692 if (next == len) {
693 raise_errmsg("Unterminated string starting at", pystr, begin);
694 goto bail;
696 c = buf[next];
697 if (c != 'u') {
698 /* Non-unicode backslash escapes */
699 end = next + 1;
700 switch (c) {
701 case '"': break;
702 case '\\': break;
703 case '/': break;
704 case 'b': c = '\b'; break;
705 case 'f': c = '\f'; break;
706 case 'n': c = '\n'; break;
707 case 'r': c = '\r'; break;
708 case 't': c = '\t'; break;
709 default: c = 0;
711 if (c == 0) {
712 raise_errmsg("Invalid \\escape", pystr, end - 2);
713 goto bail;
716 else {
717 c = 0;
718 next++;
719 end = next + 4;
720 if (end >= len) {
721 raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
722 goto bail;
724 /* Decode 4 hex digits */
725 for (; next < end; next++) {
726 Py_UNICODE digit = buf[next];
727 c <<= 4;
728 switch (digit) {
729 case '0': case '1': case '2': case '3': case '4':
730 case '5': case '6': case '7': case '8': case '9':
731 c |= (digit - '0'); break;
732 case 'a': case 'b': case 'c': case 'd': case 'e':
733 case 'f':
734 c |= (digit - 'a' + 10); break;
735 case 'A': case 'B': case 'C': case 'D': case 'E':
736 case 'F':
737 c |= (digit - 'A' + 10); break;
738 default:
739 raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
740 goto bail;
743 #ifdef Py_UNICODE_WIDE
744 /* Surrogate pair */
745 if ((c & 0xfc00) == 0xd800) {
746 Py_UNICODE c2 = 0;
747 if (end + 6 >= len) {
748 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
749 goto bail;
751 if (buf[next++] != '\\' || buf[next++] != 'u') {
752 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
753 goto bail;
755 end += 6;
756 /* Decode 4 hex digits */
757 for (; next < end; next++) {
758 c2 <<= 4;
759 Py_UNICODE digit = buf[next];
760 switch (digit) {
761 case '0': case '1': case '2': case '3': case '4':
762 case '5': case '6': case '7': case '8': case '9':
763 c2 |= (digit - '0'); break;
764 case 'a': case 'b': case 'c': case 'd': case 'e':
765 case 'f':
766 c2 |= (digit - 'a' + 10); break;
767 case 'A': case 'B': case 'C': case 'D': case 'E':
768 case 'F':
769 c2 |= (digit - 'A' + 10); break;
770 default:
771 raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
772 goto bail;
775 if ((c2 & 0xfc00) != 0xdc00) {
776 raise_errmsg("Unpaired high surrogate", pystr, end - 5);
777 goto bail;
779 c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
781 else if ((c & 0xfc00) == 0xdc00) {
782 raise_errmsg("Unpaired low surrogate", pystr, end - 5);
783 goto bail;
785 #endif
787 chunk = PyUnicode_FromUnicode(&c, 1);
788 if (chunk == NULL) {
789 goto bail;
791 if (PyList_Append(chunks, chunk)) {
792 Py_DECREF(chunk);
793 goto bail;
795 Py_DECREF(chunk);
798 rval = join_list_unicode(chunks);
799 if (rval == NULL) {
800 goto bail;
802 Py_DECREF(chunks);
803 *next_end_ptr = end;
804 return rval;
805 bail:
806 *next_end_ptr = -1;
807 Py_XDECREF(chunks);
808 return NULL;
811 PyDoc_STRVAR(pydoc_scanstring,
812 "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n"
813 "\n"
814 "Scan the string s for a JSON string. End is the index of the\n"
815 "character in s after the quote that started the JSON string.\n"
816 "Unescapes all valid JSON string escape sequences and raises ValueError\n"
817 "on attempt to decode an invalid string. If strict is False then literal\n"
818 "control characters are allowed in the string.\n"
819 "\n"
820 "Returns a tuple of the decoded string and the index of the character in s\n"
821 "after the end quote."
824 static PyObject *
825 py_scanstring(PyObject* self UNUSED, PyObject *args)
827 PyObject *pystr;
828 PyObject *rval;
829 Py_ssize_t end;
830 Py_ssize_t next_end = -1;
831 char *encoding = NULL;
832 int strict = 1;
833 if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) {
834 return NULL;
836 if (encoding == NULL) {
837 encoding = DEFAULT_ENCODING;
839 if (PyString_Check(pystr)) {
840 rval = scanstring_str(pystr, end, encoding, strict, &next_end);
842 else if (PyUnicode_Check(pystr)) {
843 rval = scanstring_unicode(pystr, end, strict, &next_end);
845 else {
846 PyErr_Format(PyExc_TypeError,
847 "first argument must be a string, not %.80s",
848 Py_TYPE(pystr)->tp_name);
849 return NULL;
851 return _build_rval_index_tuple(rval, next_end);
854 PyDoc_STRVAR(pydoc_encode_basestring_ascii,
855 "encode_basestring_ascii(basestring) -> str\n"
856 "\n"
857 "Return an ASCII-only JSON representation of a Python string"
860 static PyObject *
861 py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr)
863 /* Return an ASCII-only JSON representation of a Python string */
864 /* METH_O */
865 if (PyString_Check(pystr)) {
866 return ascii_escape_str(pystr);
868 else if (PyUnicode_Check(pystr)) {
869 return ascii_escape_unicode(pystr);
871 else {
872 PyErr_Format(PyExc_TypeError,
873 "first argument must be a string, not %.80s",
874 Py_TYPE(pystr)->tp_name);
875 return NULL;
879 static void
880 scanner_dealloc(PyObject *self)
882 /* Deallocate scanner object */
883 scanner_clear(self);
884 Py_TYPE(self)->tp_free(self);
887 static int
888 scanner_traverse(PyObject *self, visitproc visit, void *arg)
890 PyScannerObject *s;
891 assert(PyScanner_Check(self));
892 s = (PyScannerObject *)self;
893 Py_VISIT(s->encoding);
894 Py_VISIT(s->strict);
895 Py_VISIT(s->object_hook);
896 Py_VISIT(s->pairs_hook);
897 Py_VISIT(s->parse_float);
898 Py_VISIT(s->parse_int);
899 Py_VISIT(s->parse_constant);
900 return 0;
903 static int
904 scanner_clear(PyObject *self)
906 PyScannerObject *s;
907 assert(PyScanner_Check(self));
908 s = (PyScannerObject *)self;
909 Py_CLEAR(s->encoding);
910 Py_CLEAR(s->strict);
911 Py_CLEAR(s->object_hook);
912 Py_CLEAR(s->pairs_hook);
913 Py_CLEAR(s->parse_float);
914 Py_CLEAR(s->parse_int);
915 Py_CLEAR(s->parse_constant);
916 return 0;
919 static PyObject *
920 _parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
921 /* Read a JSON object from PyString pystr.
922 idx is the index of the first character after the opening curly brace.
923 *next_idx_ptr is a return-by-reference index to the first character after
924 the closing curly brace.
926 Returns a new PyObject (usually a dict, but object_hook can change that)
928 char *str = PyString_AS_STRING(pystr);
929 Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
930 PyObject *rval;
931 PyObject *pairs;
932 PyObject *item;
933 PyObject *key = NULL;
934 PyObject *val = NULL;
935 char *encoding = PyString_AS_STRING(s->encoding);
936 int strict = PyObject_IsTrue(s->strict);
937 Py_ssize_t next_idx;
939 pairs = PyList_New(0);
940 if (pairs == NULL)
941 return NULL;
943 /* skip whitespace after { */
944 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
946 /* only loop if the object is non-empty */
947 if (idx <= end_idx && str[idx] != '}') {
948 while (idx <= end_idx) {
949 /* read key */
950 if (str[idx] != '"') {
951 raise_errmsg("Expecting property name", pystr, idx);
952 goto bail;
954 key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx);
955 if (key == NULL)
956 goto bail;
957 idx = next_idx;
959 /* skip whitespace between key and : delimiter, read :, skip whitespace */
960 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
961 if (idx > end_idx || str[idx] != ':') {
962 raise_errmsg("Expecting : delimiter", pystr, idx);
963 goto bail;
965 idx++;
966 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
968 /* read any JSON data type */
969 val = scan_once_str(s, pystr, idx, &next_idx);
970 if (val == NULL)
971 goto bail;
973 item = PyTuple_Pack(2, key, val);
974 if (item == NULL)
975 goto bail;
976 Py_CLEAR(key);
977 Py_CLEAR(val);
978 if (PyList_Append(pairs, item) == -1) {
979 Py_DECREF(item);
980 goto bail;
982 Py_DECREF(item);
983 idx = next_idx;
985 /* skip whitespace before } or , */
986 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
988 /* bail if the object is closed or we didn't get the , delimiter */
989 if (idx > end_idx) break;
990 if (str[idx] == '}') {
991 break;
993 else if (str[idx] != ',') {
994 raise_errmsg("Expecting , delimiter", pystr, idx);
995 goto bail;
997 idx++;
999 /* skip whitespace after , delimiter */
1000 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1003 /* verify that idx < end_idx, str[idx] should be '}' */
1004 if (idx > end_idx || str[idx] != '}') {
1005 raise_errmsg("Expecting object", pystr, end_idx);
1006 goto bail;
1009 /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */
1010 if (s->pairs_hook != Py_None) {
1011 val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL);
1012 if (val == NULL)
1013 goto bail;
1014 Py_DECREF(pairs);
1015 *next_idx_ptr = idx + 1;
1016 return val;
1019 rval = PyObject_CallFunctionObjArgs((PyObject *)(&PyDict_Type),
1020 pairs, NULL);
1021 if (rval == NULL)
1022 goto bail;
1023 Py_CLEAR(pairs);
1025 /* if object_hook is not None: rval = object_hook(rval) */
1026 if (s->object_hook != Py_None) {
1027 val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
1028 if (val == NULL)
1029 goto bail;
1030 Py_DECREF(rval);
1031 rval = val;
1032 val = NULL;
1034 *next_idx_ptr = idx + 1;
1035 return rval;
1036 bail:
1037 Py_XDECREF(key);
1038 Py_XDECREF(val);
1039 Py_XDECREF(pairs);
1040 return NULL;
1043 static PyObject *
1044 _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
1045 /* Read a JSON object from PyUnicode pystr.
1046 idx is the index of the first character after the opening curly brace.
1047 *next_idx_ptr is a return-by-reference index to the first character after
1048 the closing curly brace.
1050 Returns a new PyObject (usually a dict, but object_hook can change that)
1052 Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
1053 Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
1054 PyObject *rval;
1055 PyObject *pairs;
1056 PyObject *item;
1057 PyObject *key = NULL;
1058 PyObject *val = NULL;
1059 int strict = PyObject_IsTrue(s->strict);
1060 Py_ssize_t next_idx;
1062 pairs = PyList_New(0);
1063 if (pairs == NULL)
1064 return NULL;
1066 /* skip whitespace after { */
1067 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1069 /* only loop if the object is non-empty */
1070 if (idx <= end_idx && str[idx] != '}') {
1071 while (idx <= end_idx) {
1072 /* read key */
1073 if (str[idx] != '"') {
1074 raise_errmsg("Expecting property name", pystr, idx);
1075 goto bail;
1077 key = scanstring_unicode(pystr, idx + 1, strict, &next_idx);
1078 if (key == NULL)
1079 goto bail;
1080 idx = next_idx;
1082 /* skip whitespace between key and : delimiter, read :, skip whitespace */
1083 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1084 if (idx > end_idx || str[idx] != ':') {
1085 raise_errmsg("Expecting : delimiter", pystr, idx);
1086 goto bail;
1088 idx++;
1089 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1091 /* read any JSON term */
1092 val = scan_once_unicode(s, pystr, idx, &next_idx);
1093 if (val == NULL)
1094 goto bail;
1096 item = PyTuple_Pack(2, key, val);
1097 if (item == NULL)
1098 goto bail;
1099 Py_CLEAR(key);
1100 Py_CLEAR(val);
1101 if (PyList_Append(pairs, item) == -1) {
1102 Py_DECREF(item);
1103 goto bail;
1105 Py_DECREF(item);
1106 idx = next_idx;
1108 /* skip whitespace before } or , */
1109 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1111 /* bail if the object is closed or we didn't get the , delimiter */
1112 if (idx > end_idx) break;
1113 if (str[idx] == '}') {
1114 break;
1116 else if (str[idx] != ',') {
1117 raise_errmsg("Expecting , delimiter", pystr, idx);
1118 goto bail;
1120 idx++;
1122 /* skip whitespace after , delimiter */
1123 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1127 /* verify that idx < end_idx, str[idx] should be '}' */
1128 if (idx > end_idx || str[idx] != '}') {
1129 raise_errmsg("Expecting object", pystr, end_idx);
1130 goto bail;
1133 /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */
1134 if (s->pairs_hook != Py_None) {
1135 val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL);
1136 if (val == NULL)
1137 goto bail;
1138 Py_DECREF(pairs);
1139 *next_idx_ptr = idx + 1;
1140 return val;
1143 rval = PyObject_CallFunctionObjArgs((PyObject *)(&PyDict_Type),
1144 pairs, NULL);
1145 if (rval == NULL)
1146 goto bail;
1147 Py_CLEAR(pairs);
1149 /* if object_hook is not None: rval = object_hook(rval) */
1150 if (s->object_hook != Py_None) {
1151 val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
1152 if (val == NULL)
1153 goto bail;
1154 Py_DECREF(rval);
1155 rval = val;
1156 val = NULL;
1158 *next_idx_ptr = idx + 1;
1159 return rval;
1160 bail:
1161 Py_XDECREF(key);
1162 Py_XDECREF(val);
1163 Py_XDECREF(pairs);
1164 return NULL;
1167 static PyObject *
1168 _parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
1169 /* Read a JSON array from PyString pystr.
1170 idx is the index of the first character after the opening brace.
1171 *next_idx_ptr is a return-by-reference index to the first character after
1172 the closing brace.
1174 Returns a new PyList
1176 char *str = PyString_AS_STRING(pystr);
1177 Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
1178 PyObject *val = NULL;
1179 PyObject *rval = PyList_New(0);
1180 Py_ssize_t next_idx;
1181 if (rval == NULL)
1182 return NULL;
1184 /* skip whitespace after [ */
1185 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1187 /* only loop if the array is non-empty */
1188 if (idx <= end_idx && str[idx] != ']') {
1189 while (idx <= end_idx) {
1191 /* read any JSON term and de-tuplefy the (rval, idx) */
1192 val = scan_once_str(s, pystr, idx, &next_idx);
1193 if (val == NULL)
1194 goto bail;
1196 if (PyList_Append(rval, val) == -1)
1197 goto bail;
1199 Py_CLEAR(val);
1200 idx = next_idx;
1202 /* skip whitespace between term and , */
1203 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1205 /* bail if the array is closed or we didn't get the , delimiter */
1206 if (idx > end_idx) break;
1207 if (str[idx] == ']') {
1208 break;
1210 else if (str[idx] != ',') {
1211 raise_errmsg("Expecting , delimiter", pystr, idx);
1212 goto bail;
1214 idx++;
1216 /* skip whitespace after , */
1217 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1221 /* verify that idx < end_idx, str[idx] should be ']' */
1222 if (idx > end_idx || str[idx] != ']') {
1223 raise_errmsg("Expecting object", pystr, end_idx);
1224 goto bail;
1226 *next_idx_ptr = idx + 1;
1227 return rval;
1228 bail:
1229 Py_XDECREF(val);
1230 Py_DECREF(rval);
1231 return NULL;
1234 static PyObject *
1235 _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
1236 /* Read a JSON array from PyString pystr.
1237 idx is the index of the first character after the opening brace.
1238 *next_idx_ptr is a return-by-reference index to the first character after
1239 the closing brace.
1241 Returns a new PyList
1243 Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
1244 Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
1245 PyObject *val = NULL;
1246 PyObject *rval = PyList_New(0);
1247 Py_ssize_t next_idx;
1248 if (rval == NULL)
1249 return NULL;
1251 /* skip whitespace after [ */
1252 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1254 /* only loop if the array is non-empty */
1255 if (idx <= end_idx && str[idx] != ']') {
1256 while (idx <= end_idx) {
1258 /* read any JSON term */
1259 val = scan_once_unicode(s, pystr, idx, &next_idx);
1260 if (val == NULL)
1261 goto bail;
1263 if (PyList_Append(rval, val) == -1)
1264 goto bail;
1266 Py_CLEAR(val);
1267 idx = next_idx;
1269 /* skip whitespace between term and , */
1270 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1272 /* bail if the array is closed or we didn't get the , delimiter */
1273 if (idx > end_idx) break;
1274 if (str[idx] == ']') {
1275 break;
1277 else if (str[idx] != ',') {
1278 raise_errmsg("Expecting , delimiter", pystr, idx);
1279 goto bail;
1281 idx++;
1283 /* skip whitespace after , */
1284 while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
1288 /* verify that idx < end_idx, str[idx] should be ']' */
1289 if (idx > end_idx || str[idx] != ']') {
1290 raise_errmsg("Expecting object", pystr, end_idx);
1291 goto bail;
1293 *next_idx_ptr = idx + 1;
1294 return rval;
1295 bail:
1296 Py_XDECREF(val);
1297 Py_DECREF(rval);
1298 return NULL;
1301 static PyObject *
1302 _parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
1303 /* Read a JSON constant from PyString pystr.
1304 constant is the constant string that was found
1305 ("NaN", "Infinity", "-Infinity").
1306 idx is the index of the first character of the constant
1307 *next_idx_ptr is a return-by-reference index to the first character after
1308 the constant.
1310 Returns the result of parse_constant
1312 PyObject *cstr;
1313 PyObject *rval;
1314 /* constant is "NaN", "Infinity", or "-Infinity" */
1315 cstr = PyString_InternFromString(constant);
1316 if (cstr == NULL)
1317 return NULL;
1319 /* rval = parse_constant(constant) */
1320 rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL);
1321 idx += PyString_GET_SIZE(cstr);
1322 Py_DECREF(cstr);
1323 *next_idx_ptr = idx;
1324 return rval;
1327 static PyObject *
1328 _match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) {
1329 /* Read a JSON number from PyString pystr.
1330 idx is the index of the first character of the number
1331 *next_idx_ptr is a return-by-reference index to the first character after
1332 the number.
1334 Returns a new PyObject representation of that number:
1335 PyInt, PyLong, or PyFloat.
1336 May return other types if parse_int or parse_float are set
1338 char *str = PyString_AS_STRING(pystr);
1339 Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
1340 Py_ssize_t idx = start;
1341 int is_float = 0;
1342 PyObject *rval;
1343 PyObject *numstr;
1345 /* read a sign if it's there, make sure it's not the end of the string */
1346 if (str[idx] == '-') {
1347 idx++;
1348 if (idx > end_idx) {
1349 PyErr_SetNone(PyExc_StopIteration);
1350 return NULL;
1354 /* read as many integer digits as we find as long as it doesn't start with 0 */
1355 if (str[idx] >= '1' && str[idx] <= '9') {
1356 idx++;
1357 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1359 /* if it starts with 0 we only expect one integer digit */
1360 else if (str[idx] == '0') {
1361 idx++;
1363 /* no integer digits, error */
1364 else {
1365 PyErr_SetNone(PyExc_StopIteration);
1366 return NULL;
1369 /* if the next char is '.' followed by a digit then read all float digits */
1370 if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') {
1371 is_float = 1;
1372 idx += 2;
1373 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1376 /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */
1377 if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) {
1379 /* save the index of the 'e' or 'E' just in case we need to backtrack */
1380 Py_ssize_t e_start = idx;
1381 idx++;
1383 /* read an exponent sign if present */
1384 if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++;
1386 /* read all digits */
1387 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1389 /* if we got a digit, then parse as float. if not, backtrack */
1390 if (str[idx - 1] >= '0' && str[idx - 1] <= '9') {
1391 is_float = 1;
1393 else {
1394 idx = e_start;
1398 /* copy the section we determined to be a number */
1399 numstr = PyString_FromStringAndSize(&str[start], idx - start);
1400 if (numstr == NULL)
1401 return NULL;
1402 if (is_float) {
1403 /* parse as a float using a fast path if available, otherwise call user defined method */
1404 if (s->parse_float != (PyObject *)&PyFloat_Type) {
1405 rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
1407 else {
1408 rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr)));
1411 else {
1412 /* parse as an int using a fast path if available, otherwise call user defined method */
1413 if (s->parse_int != (PyObject *)&PyInt_Type) {
1414 rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
1416 else {
1417 rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10);
1420 Py_DECREF(numstr);
1421 *next_idx_ptr = idx;
1422 return rval;
1425 static PyObject *
1426 _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) {
1427 /* Read a JSON number from PyUnicode pystr.
1428 idx is the index of the first character of the number
1429 *next_idx_ptr is a return-by-reference index to the first character after
1430 the number.
1432 Returns a new PyObject representation of that number:
1433 PyInt, PyLong, or PyFloat.
1434 May return other types if parse_int or parse_float are set
1436 Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
1437 Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
1438 Py_ssize_t idx = start;
1439 int is_float = 0;
1440 PyObject *rval;
1441 PyObject *numstr;
1443 /* read a sign if it's there, make sure it's not the end of the string */
1444 if (str[idx] == '-') {
1445 idx++;
1446 if (idx > end_idx) {
1447 PyErr_SetNone(PyExc_StopIteration);
1448 return NULL;
1452 /* read as many integer digits as we find as long as it doesn't start with 0 */
1453 if (str[idx] >= '1' && str[idx] <= '9') {
1454 idx++;
1455 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1457 /* if it starts with 0 we only expect one integer digit */
1458 else if (str[idx] == '0') {
1459 idx++;
1461 /* no integer digits, error */
1462 else {
1463 PyErr_SetNone(PyExc_StopIteration);
1464 return NULL;
1467 /* if the next char is '.' followed by a digit then read all float digits */
1468 if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') {
1469 is_float = 1;
1470 idx += 2;
1471 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1474 /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */
1475 if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) {
1476 Py_ssize_t e_start = idx;
1477 idx++;
1479 /* read an exponent sign if present */
1480 if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++;
1482 /* read all digits */
1483 while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
1485 /* if we got a digit, then parse as float. if not, backtrack */
1486 if (str[idx - 1] >= '0' && str[idx - 1] <= '9') {
1487 is_float = 1;
1489 else {
1490 idx = e_start;
1494 /* copy the section we determined to be a number */
1495 numstr = PyUnicode_FromUnicode(&str[start], idx - start);
1496 if (numstr == NULL)
1497 return NULL;
1498 if (is_float) {
1499 /* parse as a float using a fast path if available, otherwise call user defined method */
1500 if (s->parse_float != (PyObject *)&PyFloat_Type) {
1501 rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
1503 else {
1504 rval = PyFloat_FromString(numstr, NULL);
1507 else {
1508 /* no fast path for unicode -> int, just call */
1509 rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
1511 Py_DECREF(numstr);
1512 *next_idx_ptr = idx;
1513 return rval;
1516 static PyObject *
1517 scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
1519 /* Read one JSON term (of any kind) from PyString pystr.
1520 idx is the index of the first character of the term
1521 *next_idx_ptr is a return-by-reference index to the first character after
1522 the number.
1524 Returns a new PyObject representation of the term.
1526 char *str = PyString_AS_STRING(pystr);
1527 Py_ssize_t length = PyString_GET_SIZE(pystr);
1528 if (idx >= length) {
1529 PyErr_SetNone(PyExc_StopIteration);
1530 return NULL;
1532 switch (str[idx]) {
1533 case '"':
1534 /* string */
1535 return scanstring_str(pystr, idx + 1,
1536 PyString_AS_STRING(s->encoding),
1537 PyObject_IsTrue(s->strict),
1538 next_idx_ptr);
1539 case '{':
1540 /* object */
1541 return _parse_object_str(s, pystr, idx + 1, next_idx_ptr);
1542 case '[':
1543 /* array */
1544 return _parse_array_str(s, pystr, idx + 1, next_idx_ptr);
1545 case 'n':
1546 /* null */
1547 if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') {
1548 Py_INCREF(Py_None);
1549 *next_idx_ptr = idx + 4;
1550 return Py_None;
1552 break;
1553 case 't':
1554 /* true */
1555 if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') {
1556 Py_INCREF(Py_True);
1557 *next_idx_ptr = idx + 4;
1558 return Py_True;
1560 break;
1561 case 'f':
1562 /* false */
1563 if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') {
1564 Py_INCREF(Py_False);
1565 *next_idx_ptr = idx + 5;
1566 return Py_False;
1568 break;
1569 case 'N':
1570 /* NaN */
1571 if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') {
1572 return _parse_constant(s, "NaN", idx, next_idx_ptr);
1574 break;
1575 case 'I':
1576 /* Infinity */
1577 if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') {
1578 return _parse_constant(s, "Infinity", idx, next_idx_ptr);
1580 break;
1581 case '-':
1582 /* -Infinity */
1583 if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') {
1584 return _parse_constant(s, "-Infinity", idx, next_idx_ptr);
1586 break;
1588 /* Didn't find a string, object, array, or named constant. Look for a number. */
1589 return _match_number_str(s, pystr, idx, next_idx_ptr);
1592 static PyObject *
1593 scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
1595 /* Read one JSON term (of any kind) from PyUnicode pystr.
1596 idx is the index of the first character of the term
1597 *next_idx_ptr is a return-by-reference index to the first character after
1598 the number.
1600 Returns a new PyObject representation of the term.
1602 Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
1603 Py_ssize_t length = PyUnicode_GET_SIZE(pystr);
1604 if (idx >= length) {
1605 PyErr_SetNone(PyExc_StopIteration);
1606 return NULL;
1608 switch (str[idx]) {
1609 case '"':
1610 /* string */
1611 return scanstring_unicode(pystr, idx + 1,
1612 PyObject_IsTrue(s->strict),
1613 next_idx_ptr);
1614 case '{':
1615 /* object */
1616 return _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr);
1617 case '[':
1618 /* array */
1619 return _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr);
1620 case 'n':
1621 /* null */
1622 if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') {
1623 Py_INCREF(Py_None);
1624 *next_idx_ptr = idx + 4;
1625 return Py_None;
1627 break;
1628 case 't':
1629 /* true */
1630 if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') {
1631 Py_INCREF(Py_True);
1632 *next_idx_ptr = idx + 4;
1633 return Py_True;
1635 break;
1636 case 'f':
1637 /* false */
1638 if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') {
1639 Py_INCREF(Py_False);
1640 *next_idx_ptr = idx + 5;
1641 return Py_False;
1643 break;
1644 case 'N':
1645 /* NaN */
1646 if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') {
1647 return _parse_constant(s, "NaN", idx, next_idx_ptr);
1649 break;
1650 case 'I':
1651 /* Infinity */
1652 if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') {
1653 return _parse_constant(s, "Infinity", idx, next_idx_ptr);
1655 break;
1656 case '-':
1657 /* -Infinity */
1658 if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') {
1659 return _parse_constant(s, "-Infinity", idx, next_idx_ptr);
1661 break;
1663 /* Didn't find a string, object, array, or named constant. Look for a number. */
1664 return _match_number_unicode(s, pystr, idx, next_idx_ptr);
1667 static PyObject *
1668 scanner_call(PyObject *self, PyObject *args, PyObject *kwds)
1670 /* Python callable interface to scan_once_{str,unicode} */
1671 PyObject *pystr;
1672 PyObject *rval;
1673 Py_ssize_t idx;
1674 Py_ssize_t next_idx = -1;
1675 static char *kwlist[] = {"string", "idx", NULL};
1676 PyScannerObject *s;
1677 assert(PyScanner_Check(self));
1678 s = (PyScannerObject *)self;
1679 if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx))
1680 return NULL;
1682 if (PyString_Check(pystr)) {
1683 rval = scan_once_str(s, pystr, idx, &next_idx);
1685 else if (PyUnicode_Check(pystr)) {
1686 rval = scan_once_unicode(s, pystr, idx, &next_idx);
1688 else {
1689 PyErr_Format(PyExc_TypeError,
1690 "first argument must be a string, not %.80s",
1691 Py_TYPE(pystr)->tp_name);
1692 return NULL;
1694 return _build_rval_index_tuple(rval, next_idx);
1697 static PyObject *
1698 scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1700 PyScannerObject *s;
1701 s = (PyScannerObject *)type->tp_alloc(type, 0);
1702 if (s != NULL) {
1703 s->encoding = NULL;
1704 s->strict = NULL;
1705 s->object_hook = NULL;
1706 s->pairs_hook = NULL;
1707 s->parse_float = NULL;
1708 s->parse_int = NULL;
1709 s->parse_constant = NULL;
1711 return (PyObject *)s;
1714 static int
1715 scanner_init(PyObject *self, PyObject *args, PyObject *kwds)
1717 /* Initialize Scanner object */
1718 PyObject *ctx;
1719 static char *kwlist[] = {"context", NULL};
1720 PyScannerObject *s;
1722 assert(PyScanner_Check(self));
1723 s = (PyScannerObject *)self;
1725 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx))
1726 return -1;
1728 /* PyString_AS_STRING is used on encoding */
1729 s->encoding = PyObject_GetAttrString(ctx, "encoding");
1730 if (s->encoding == Py_None) {
1731 Py_DECREF(Py_None);
1732 s->encoding = PyString_InternFromString(DEFAULT_ENCODING);
1734 else if (PyUnicode_Check(s->encoding)) {
1735 PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL);
1736 Py_DECREF(s->encoding);
1737 s->encoding = tmp;
1739 if (s->encoding == NULL || !PyString_Check(s->encoding))
1740 goto bail;
1742 /* All of these will fail "gracefully" so we don't need to verify them */
1743 s->strict = PyObject_GetAttrString(ctx, "strict");
1744 if (s->strict == NULL)
1745 goto bail;
1746 s->object_hook = PyObject_GetAttrString(ctx, "object_hook");
1747 if (s->object_hook == NULL)
1748 goto bail;
1749 s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook");
1750 if (s->object_hook == NULL)
1751 goto bail;
1752 s->parse_float = PyObject_GetAttrString(ctx, "parse_float");
1753 if (s->parse_float == NULL)
1754 goto bail;
1755 s->parse_int = PyObject_GetAttrString(ctx, "parse_int");
1756 if (s->parse_int == NULL)
1757 goto bail;
1758 s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant");
1759 if (s->parse_constant == NULL)
1760 goto bail;
1762 return 0;
1764 bail:
1765 Py_CLEAR(s->encoding);
1766 Py_CLEAR(s->strict);
1767 Py_CLEAR(s->object_hook);
1768 Py_CLEAR(s->pairs_hook);
1769 Py_CLEAR(s->parse_float);
1770 Py_CLEAR(s->parse_int);
1771 Py_CLEAR(s->parse_constant);
1772 return -1;
1775 PyDoc_STRVAR(scanner_doc, "JSON scanner object");
1777 static
1778 PyTypeObject PyScannerType = {
1779 PyObject_HEAD_INIT(NULL)
1780 0, /* tp_internal */
1781 "_json.Scanner", /* tp_name */
1782 sizeof(PyScannerObject), /* tp_basicsize */
1783 0, /* tp_itemsize */
1784 scanner_dealloc, /* tp_dealloc */
1785 0, /* tp_print */
1786 0, /* tp_getattr */
1787 0, /* tp_setattr */
1788 0, /* tp_compare */
1789 0, /* tp_repr */
1790 0, /* tp_as_number */
1791 0, /* tp_as_sequence */
1792 0, /* tp_as_mapping */
1793 0, /* tp_hash */
1794 scanner_call, /* tp_call */
1795 0, /* tp_str */
1796 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */
1797 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */
1798 0, /* tp_as_buffer */
1799 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
1800 scanner_doc, /* tp_doc */
1801 scanner_traverse, /* tp_traverse */
1802 scanner_clear, /* tp_clear */
1803 0, /* tp_richcompare */
1804 0, /* tp_weaklistoffset */
1805 0, /* tp_iter */
1806 0, /* tp_iternext */
1807 0, /* tp_methods */
1808 scanner_members, /* tp_members */
1809 0, /* tp_getset */
1810 0, /* tp_base */
1811 0, /* tp_dict */
1812 0, /* tp_descr_get */
1813 0, /* tp_descr_set */
1814 0, /* tp_dictoffset */
1815 scanner_init, /* tp_init */
1816 0,/* PyType_GenericAlloc, */ /* tp_alloc */
1817 scanner_new, /* tp_new */
1818 0,/* PyObject_GC_Del, */ /* tp_free */
1821 static PyObject *
1822 encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1824 PyEncoderObject *s;
1825 s = (PyEncoderObject *)type->tp_alloc(type, 0);
1826 if (s != NULL) {
1827 s->markers = NULL;
1828 s->defaultfn = NULL;
1829 s->encoder = NULL;
1830 s->indent = NULL;
1831 s->key_separator = NULL;
1832 s->item_separator = NULL;
1833 s->sort_keys = NULL;
1834 s->skipkeys = NULL;
1836 return (PyObject *)s;
1839 static int
1840 encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1842 /* initialize Encoder object */
1843 static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL};
1845 PyEncoderObject *s;
1846 PyObject *allow_nan;
1848 assert(PyEncoder_Check(self));
1849 s = (PyEncoderObject *)self;
1851 if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOO:make_encoder", kwlist,
1852 &s->markers, &s->defaultfn, &s->encoder, &s->indent, &s->key_separator, &s->item_separator, &s->sort_keys, &s->skipkeys, &allow_nan))
1853 return -1;
1855 Py_INCREF(s->markers);
1856 Py_INCREF(s->defaultfn);
1857 Py_INCREF(s->encoder);
1858 Py_INCREF(s->indent);
1859 Py_INCREF(s->key_separator);
1860 Py_INCREF(s->item_separator);
1861 Py_INCREF(s->sort_keys);
1862 Py_INCREF(s->skipkeys);
1863 s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii);
1864 s->allow_nan = PyObject_IsTrue(allow_nan);
1865 return 0;
1868 static PyObject *
1869 encoder_call(PyObject *self, PyObject *args, PyObject *kwds)
1871 /* Python callable interface to encode_listencode_obj */
1872 static char *kwlist[] = {"obj", "_current_indent_level", NULL};
1873 PyObject *obj;
1874 PyObject *rval;
1875 Py_ssize_t indent_level;
1876 PyEncoderObject *s;
1877 assert(PyEncoder_Check(self));
1878 s = (PyEncoderObject *)self;
1879 if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist,
1880 &obj, _convertPyInt_AsSsize_t, &indent_level))
1881 return NULL;
1882 rval = PyList_New(0);
1883 if (rval == NULL)
1884 return NULL;
1885 if (encoder_listencode_obj(s, rval, obj, indent_level)) {
1886 Py_DECREF(rval);
1887 return NULL;
1889 return rval;
1892 static PyObject *
1893 _encoded_const(PyObject *obj)
1895 /* Return the JSON string representation of None, True, False */
1896 if (obj == Py_None) {
1897 static PyObject *s_null = NULL;
1898 if (s_null == NULL) {
1899 s_null = PyString_InternFromString("null");
1901 Py_INCREF(s_null);
1902 return s_null;
1904 else if (obj == Py_True) {
1905 static PyObject *s_true = NULL;
1906 if (s_true == NULL) {
1907 s_true = PyString_InternFromString("true");
1909 Py_INCREF(s_true);
1910 return s_true;
1912 else if (obj == Py_False) {
1913 static PyObject *s_false = NULL;
1914 if (s_false == NULL) {
1915 s_false = PyString_InternFromString("false");
1917 Py_INCREF(s_false);
1918 return s_false;
1920 else {
1921 PyErr_SetString(PyExc_ValueError, "not a const");
1922 return NULL;
1926 static PyObject *
1927 encoder_encode_float(PyEncoderObject *s, PyObject *obj)
1929 /* Return the JSON representation of a PyFloat */
1930 double i = PyFloat_AS_DOUBLE(obj);
1931 if (!Py_IS_FINITE(i)) {
1932 if (!s->allow_nan) {
1933 PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant");
1934 return NULL;
1936 if (i > 0) {
1937 return PyString_FromString("Infinity");
1939 else if (i < 0) {
1940 return PyString_FromString("-Infinity");
1942 else {
1943 return PyString_FromString("NaN");
1946 /* Use a better float format here? */
1947 return PyObject_Repr(obj);
1950 static PyObject *
1951 encoder_encode_string(PyEncoderObject *s, PyObject *obj)
1953 /* Return the JSON representation of a string */
1954 if (s->fast_encode)
1955 return py_encode_basestring_ascii(NULL, obj);
1956 else
1957 return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL);
1960 static int
1961 _steal_list_append(PyObject *lst, PyObject *stolen)
1963 /* Append stolen and then decrement its reference count */
1964 int rval = PyList_Append(lst, stolen);
1965 Py_DECREF(stolen);
1966 return rval;
1969 static int
1970 encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level)
1972 /* Encode Python object obj to a JSON term, rval is a PyList */
1973 PyObject *newobj;
1974 int rv;
1976 if (obj == Py_None || obj == Py_True || obj == Py_False) {
1977 PyObject *cstr = _encoded_const(obj);
1978 if (cstr == NULL)
1979 return -1;
1980 return _steal_list_append(rval, cstr);
1982 else if (PyString_Check(obj) || PyUnicode_Check(obj))
1984 PyObject *encoded = encoder_encode_string(s, obj);
1985 if (encoded == NULL)
1986 return -1;
1987 return _steal_list_append(rval, encoded);
1989 else if (PyInt_Check(obj) || PyLong_Check(obj)) {
1990 PyObject *encoded = PyObject_Str(obj);
1991 if (encoded == NULL)
1992 return -1;
1993 return _steal_list_append(rval, encoded);
1995 else if (PyFloat_Check(obj)) {
1996 PyObject *encoded = encoder_encode_float(s, obj);
1997 if (encoded == NULL)
1998 return -1;
1999 return _steal_list_append(rval, encoded);
2001 else if (PyList_Check(obj) || PyTuple_Check(obj)) {
2002 return encoder_listencode_list(s, rval, obj, indent_level);
2004 else if (PyDict_Check(obj)) {
2005 return encoder_listencode_dict(s, rval, obj, indent_level);
2007 else {
2008 PyObject *ident = NULL;
2009 if (s->markers != Py_None) {
2010 int has_key;
2011 ident = PyLong_FromVoidPtr(obj);
2012 if (ident == NULL)
2013 return -1;
2014 has_key = PyDict_Contains(s->markers, ident);
2015 if (has_key) {
2016 if (has_key != -1)
2017 PyErr_SetString(PyExc_ValueError, "Circular reference detected");
2018 Py_DECREF(ident);
2019 return -1;
2021 if (PyDict_SetItem(s->markers, ident, obj)) {
2022 Py_DECREF(ident);
2023 return -1;
2026 newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL);
2027 if (newobj == NULL) {
2028 Py_XDECREF(ident);
2029 return -1;
2031 rv = encoder_listencode_obj(s, rval, newobj, indent_level);
2032 Py_DECREF(newobj);
2033 if (rv) {
2034 Py_XDECREF(ident);
2035 return -1;
2037 if (ident != NULL) {
2038 if (PyDict_DelItem(s->markers, ident)) {
2039 Py_XDECREF(ident);
2040 return -1;
2042 Py_XDECREF(ident);
2044 return rv;
2048 static int
2049 encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level)
2051 /* Encode Python dict dct a JSON term, rval is a PyList */
2052 static PyObject *open_dict = NULL;
2053 static PyObject *close_dict = NULL;
2054 static PyObject *empty_dict = NULL;
2055 PyObject *kstr = NULL;
2056 PyObject *ident = NULL;
2057 PyObject *key, *value;
2058 Py_ssize_t pos;
2059 int skipkeys;
2060 Py_ssize_t idx;
2062 if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) {
2063 open_dict = PyString_InternFromString("{");
2064 close_dict = PyString_InternFromString("}");
2065 empty_dict = PyString_InternFromString("{}");
2066 if (open_dict == NULL || close_dict == NULL || empty_dict == NULL)
2067 return -1;
2069 if (PyDict_Size(dct) == 0)
2070 return PyList_Append(rval, empty_dict);
2072 if (s->markers != Py_None) {
2073 int has_key;
2074 ident = PyLong_FromVoidPtr(dct);
2075 if (ident == NULL)
2076 goto bail;
2077 has_key = PyDict_Contains(s->markers, ident);
2078 if (has_key) {
2079 if (has_key != -1)
2080 PyErr_SetString(PyExc_ValueError, "Circular reference detected");
2081 goto bail;
2083 if (PyDict_SetItem(s->markers, ident, dct)) {
2084 goto bail;
2088 if (PyList_Append(rval, open_dict))
2089 goto bail;
2091 if (s->indent != Py_None) {
2092 /* TODO: DOES NOT RUN */
2093 indent_level += 1;
2095 newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
2096 separator = _item_separator + newline_indent
2097 buf += newline_indent
2101 /* TODO: C speedup not implemented for sort_keys */
2103 pos = 0;
2104 skipkeys = PyObject_IsTrue(s->skipkeys);
2105 idx = 0;
2106 while (PyDict_Next(dct, &pos, &key, &value)) {
2107 PyObject *encoded;
2109 if (PyString_Check(key) || PyUnicode_Check(key)) {
2110 Py_INCREF(key);
2111 kstr = key;
2113 else if (PyFloat_Check(key)) {
2114 kstr = encoder_encode_float(s, key);
2115 if (kstr == NULL)
2116 goto bail;
2118 else if (PyInt_Check(key) || PyLong_Check(key)) {
2119 kstr = PyObject_Str(key);
2120 if (kstr == NULL)
2121 goto bail;
2123 else if (key == Py_True || key == Py_False || key == Py_None) {
2124 kstr = _encoded_const(key);
2125 if (kstr == NULL)
2126 goto bail;
2128 else if (skipkeys) {
2129 continue;
2131 else {
2132 /* TODO: include repr of key */
2133 PyErr_SetString(PyExc_ValueError, "keys must be a string");
2134 goto bail;
2137 if (idx) {
2138 if (PyList_Append(rval, s->item_separator))
2139 goto bail;
2142 encoded = encoder_encode_string(s, kstr);
2143 Py_CLEAR(kstr);
2144 if (encoded == NULL)
2145 goto bail;
2146 if (PyList_Append(rval, encoded)) {
2147 Py_DECREF(encoded);
2148 goto bail;
2150 Py_DECREF(encoded);
2151 if (PyList_Append(rval, s->key_separator))
2152 goto bail;
2153 if (encoder_listencode_obj(s, rval, value, indent_level))
2154 goto bail;
2155 idx += 1;
2157 if (ident != NULL) {
2158 if (PyDict_DelItem(s->markers, ident))
2159 goto bail;
2160 Py_CLEAR(ident);
2162 if (s->indent != Py_None) {
2163 /* TODO: DOES NOT RUN */
2164 indent_level -= 1;
2166 yield '\n' + (' ' * (_indent * _current_indent_level))
2169 if (PyList_Append(rval, close_dict))
2170 goto bail;
2171 return 0;
2173 bail:
2174 Py_XDECREF(kstr);
2175 Py_XDECREF(ident);
2176 return -1;
2180 static int
2181 encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level)
2183 /* Encode Python list seq to a JSON term, rval is a PyList */
2184 static PyObject *open_array = NULL;
2185 static PyObject *close_array = NULL;
2186 static PyObject *empty_array = NULL;
2187 PyObject *ident = NULL;
2188 PyObject *s_fast = NULL;
2189 Py_ssize_t num_items;
2190 PyObject **seq_items;
2191 Py_ssize_t i;
2193 if (open_array == NULL || close_array == NULL || empty_array == NULL) {
2194 open_array = PyString_InternFromString("[");
2195 close_array = PyString_InternFromString("]");
2196 empty_array = PyString_InternFromString("[]");
2197 if (open_array == NULL || close_array == NULL || empty_array == NULL)
2198 return -1;
2200 ident = NULL;
2201 s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence");
2202 if (s_fast == NULL)
2203 return -1;
2204 num_items = PySequence_Fast_GET_SIZE(s_fast);
2205 if (num_items == 0) {
2206 Py_DECREF(s_fast);
2207 return PyList_Append(rval, empty_array);
2210 if (s->markers != Py_None) {
2211 int has_key;
2212 ident = PyLong_FromVoidPtr(seq);
2213 if (ident == NULL)
2214 goto bail;
2215 has_key = PyDict_Contains(s->markers, ident);
2216 if (has_key) {
2217 if (has_key != -1)
2218 PyErr_SetString(PyExc_ValueError, "Circular reference detected");
2219 goto bail;
2221 if (PyDict_SetItem(s->markers, ident, seq)) {
2222 goto bail;
2226 seq_items = PySequence_Fast_ITEMS(s_fast);
2227 if (PyList_Append(rval, open_array))
2228 goto bail;
2229 if (s->indent != Py_None) {
2230 /* TODO: DOES NOT RUN */
2231 indent_level += 1;
2233 newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
2234 separator = _item_separator + newline_indent
2235 buf += newline_indent
2238 for (i = 0; i < num_items; i++) {
2239 PyObject *obj = seq_items[i];
2240 if (i) {
2241 if (PyList_Append(rval, s->item_separator))
2242 goto bail;
2244 if (encoder_listencode_obj(s, rval, obj, indent_level))
2245 goto bail;
2247 if (ident != NULL) {
2248 if (PyDict_DelItem(s->markers, ident))
2249 goto bail;
2250 Py_CLEAR(ident);
2252 if (s->indent != Py_None) {
2253 /* TODO: DOES NOT RUN */
2254 indent_level -= 1;
2256 yield '\n' + (' ' * (_indent * _current_indent_level))
2259 if (PyList_Append(rval, close_array))
2260 goto bail;
2261 Py_DECREF(s_fast);
2262 return 0;
2264 bail:
2265 Py_XDECREF(ident);
2266 Py_DECREF(s_fast);
2267 return -1;
2270 static void
2271 encoder_dealloc(PyObject *self)
2273 /* Deallocate Encoder */
2274 encoder_clear(self);
2275 Py_TYPE(self)->tp_free(self);
2278 static int
2279 encoder_traverse(PyObject *self, visitproc visit, void *arg)
2281 PyEncoderObject *s;
2282 assert(PyEncoder_Check(self));
2283 s = (PyEncoderObject *)self;
2284 Py_VISIT(s->markers);
2285 Py_VISIT(s->defaultfn);
2286 Py_VISIT(s->encoder);
2287 Py_VISIT(s->indent);
2288 Py_VISIT(s->key_separator);
2289 Py_VISIT(s->item_separator);
2290 Py_VISIT(s->sort_keys);
2291 Py_VISIT(s->skipkeys);
2292 return 0;
2295 static int
2296 encoder_clear(PyObject *self)
2298 /* Deallocate Encoder */
2299 PyEncoderObject *s;
2300 assert(PyEncoder_Check(self));
2301 s = (PyEncoderObject *)self;
2302 Py_CLEAR(s->markers);
2303 Py_CLEAR(s->defaultfn);
2304 Py_CLEAR(s->encoder);
2305 Py_CLEAR(s->indent);
2306 Py_CLEAR(s->key_separator);
2307 Py_CLEAR(s->item_separator);
2308 Py_CLEAR(s->sort_keys);
2309 Py_CLEAR(s->skipkeys);
2310 return 0;
2313 PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable");
2315 static
2316 PyTypeObject PyEncoderType = {
2317 PyObject_HEAD_INIT(NULL)
2318 0, /* tp_internal */
2319 "_json.Encoder", /* tp_name */
2320 sizeof(PyEncoderObject), /* tp_basicsize */
2321 0, /* tp_itemsize */
2322 encoder_dealloc, /* tp_dealloc */
2323 0, /* tp_print */
2324 0, /* tp_getattr */
2325 0, /* tp_setattr */
2326 0, /* tp_compare */
2327 0, /* tp_repr */
2328 0, /* tp_as_number */
2329 0, /* tp_as_sequence */
2330 0, /* tp_as_mapping */
2331 0, /* tp_hash */
2332 encoder_call, /* tp_call */
2333 0, /* tp_str */
2334 0, /* tp_getattro */
2335 0, /* tp_setattro */
2336 0, /* tp_as_buffer */
2337 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
2338 encoder_doc, /* tp_doc */
2339 encoder_traverse, /* tp_traverse */
2340 encoder_clear, /* tp_clear */
2341 0, /* tp_richcompare */
2342 0, /* tp_weaklistoffset */
2343 0, /* tp_iter */
2344 0, /* tp_iternext */
2345 0, /* tp_methods */
2346 encoder_members, /* tp_members */
2347 0, /* tp_getset */
2348 0, /* tp_base */
2349 0, /* tp_dict */
2350 0, /* tp_descr_get */
2351 0, /* tp_descr_set */
2352 0, /* tp_dictoffset */
2353 encoder_init, /* tp_init */
2354 0, /* tp_alloc */
2355 encoder_new, /* tp_new */
2356 0, /* tp_free */
2359 static PyMethodDef speedups_methods[] = {
2360 {"encode_basestring_ascii",
2361 (PyCFunction)py_encode_basestring_ascii,
2362 METH_O,
2363 pydoc_encode_basestring_ascii},
2364 {"scanstring",
2365 (PyCFunction)py_scanstring,
2366 METH_VARARGS,
2367 pydoc_scanstring},
2368 {NULL, NULL, 0, NULL}
2371 PyDoc_STRVAR(module_doc,
2372 "json speedups\n");
2374 void
2375 init_json(void)
2377 PyObject *m;
2378 PyScannerType.tp_new = PyType_GenericNew;
2379 if (PyType_Ready(&PyScannerType) < 0)
2380 return;
2381 PyEncoderType.tp_new = PyType_GenericNew;
2382 if (PyType_Ready(&PyEncoderType) < 0)
2383 return;
2384 m = Py_InitModule3("_json", speedups_methods, module_doc);
2385 Py_INCREF((PyObject*)&PyScannerType);
2386 PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType);
2387 Py_INCREF((PyObject*)&PyEncoderType);
2388 PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType);