2 /* Tokenizer implementation */
5 #include "pgenheaders.h"
10 #include "tokenizer.h"
14 #include "unicodeobject.h"
15 #include "stringobject.h"
16 #include "fileobject.h"
22 extern char *PyOS_Readline(FILE *, FILE *, char *);
23 /* Return malloc'ed string including trailing \n;
24 empty malloc'ed string for EOF;
25 NULL if interrupted */
27 /* Don't ever change this -- it would break the portability of Python code */
31 static struct tok_state
*tok_new(void);
32 static int tok_nextc(struct tok_state
*tok
);
33 static void tok_backup(struct tok_state
*tok
, int c
);
37 char *_PyParser_TokenNames
[] = {
89 /* This table must match the #defines in token.h! */
96 /* Create and initialize a new tok_state structure */
98 static struct tok_state
*
101 struct tok_state
*tok
= (struct tok_state
*)PyMem_MALLOC(
102 sizeof(struct tok_state
));
105 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= tok
->start
= NULL
;
108 tok
->tabsize
= TABSIZE
;
110 tok
->indstack
[0] = 0;
113 tok
->prompt
= tok
->nextprompt
= NULL
;
116 tok
->filename
= NULL
;
120 tok
->altindstack
[0] = 0;
121 tok
->decoding_state
= 0;
122 tok
->decoding_erred
= 0;
123 tok
->read_coding_spec
= 0;
124 tok
->encoding
= NULL
;
127 tok
->decoding_readline
= NULL
;
128 tok
->decoding_buffer
= NULL
;
136 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
138 return fgets(s
, size
, tok
->fp
);
142 decoding_feof(struct tok_state
*tok
)
144 return feof(tok
->fp
);
148 decode_str(const char *str
, struct tok_state
*tok
)
156 error_ret(struct tok_state
*tok
) /* XXX */
158 tok
->decoding_erred
= 1;
159 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
) /* see PyTokenizer_Free */
160 PyMem_FREE(tok
->buf
);
162 return NULL
; /* as if it were EOF */
166 new_string(const char *s
, Py_ssize_t len
)
168 char* result
= (char *)PyMem_MALLOC(len
+ 1);
169 if (result
!= NULL
) {
170 memcpy(result
, s
, len
);
177 get_normal_name(char *s
) /* for utf-8 and latin-1 */
181 for (i
= 0; i
< 12; i
++) {
191 if (strcmp(buf
, "utf-8") == 0 ||
192 strncmp(buf
, "utf-8-", 6) == 0)
194 else if (strcmp(buf
, "latin-1") == 0 ||
195 strcmp(buf
, "iso-8859-1") == 0 ||
196 strcmp(buf
, "iso-latin-1") == 0 ||
197 strncmp(buf
, "latin-1-", 8) == 0 ||
198 strncmp(buf
, "iso-8859-1-", 11) == 0 ||
199 strncmp(buf
, "iso-latin-1-", 12) == 0)
205 /* Return the coding spec in S, or NULL if none is found. */
208 get_coding_spec(const char *s
, Py_ssize_t size
)
211 /* Coding spec must be in a comment, and that comment must be
212 * the only statement on the source code line. */
213 for (i
= 0; i
< size
- 6; i
++) {
216 if (s
[i
] != ' ' && s
[i
] != '\t' && s
[i
] != '\014')
219 for (; i
< size
- 6; i
++) { /* XXX inefficient search */
220 const char* t
= s
+ i
;
221 if (strncmp(t
, "coding", 6) == 0) {
222 const char* begin
= NULL
;
224 if (t
[0] != ':' && t
[0] != '=')
228 } while (t
[0] == '\x20' || t
[0] == '\t');
231 while (isalnum(Py_CHARMASK(t
[0])) ||
232 t
[0] == '-' || t
[0] == '_' || t
[0] == '.')
236 char* r
= new_string(begin
, t
- begin
);
237 char* q
= get_normal_name(r
);
240 r
= new_string(q
, strlen(q
));
249 /* Check whether the line contains a coding spec. If it does,
250 invoke the set_readline function for the new encoding.
251 This function receives the tok_state and the new encoding.
252 Return 1 on success, 0 on failure. */
255 check_coding_spec(const char* line
, Py_ssize_t size
, struct tok_state
*tok
,
256 int set_readline(struct tok_state
*, const char *))
262 /* It's a continuation line, so it can't be a coding spec. */
264 cs
= get_coding_spec(line
, size
);
266 tok
->read_coding_spec
= 1;
267 if (tok
->encoding
== NULL
) {
268 assert(tok
->decoding_state
== 1); /* raw */
269 if (strcmp(cs
, "utf-8") == 0 ||
270 strcmp(cs
, "iso-8859-1") == 0) {
273 #ifdef Py_USING_UNICODE
274 r
= set_readline(tok
, cs
);
277 tok
->decoding_state
= -1;
282 /* Without Unicode support, we cannot
283 process the coding spec. Since there
284 won't be any Unicode literals, that
289 } else { /* then, compare cs with BOM */
290 r
= (strcmp(tok
->encoding
, cs
) == 0);
298 PyErr_Format(PyExc_SyntaxError
, "encoding problem: %s", cs
);
303 /* See whether the file starts with a BOM. If it does,
304 invoke the set_readline function with the new encoding.
305 Return 1 on success, 0 on failure. */
308 check_bom(int get_char(struct tok_state
*),
309 void unget_char(int, struct tok_state
*),
310 int set_readline(struct tok_state
*, const char *),
311 struct tok_state
*tok
)
313 int ch
= get_char(tok
);
314 tok
->decoding_state
= 1;
317 } else if (ch
== 0xEF) {
325 /* Disable support for UTF-16 BOMs until a decision
326 is made whether this needs to be supported. */
327 } else if (ch
== 0xFE) {
331 if (!set_readline(tok
, "utf-16-be"))
333 tok
->decoding_state
= -1;
334 } else if (ch
== 0xFF) {
338 if (!set_readline(tok
, "utf-16-le"))
340 tok
->decoding_state
= -1;
346 if (tok
->encoding
!= NULL
)
347 PyMem_FREE(tok
->encoding
);
348 tok
->encoding
= new_string("utf-8", 5); /* resulting is in utf-8 */
351 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
352 unget_char(0xFF, tok
); /* XXX this will cause a syntax error */
356 /* Read a line of text from TOK into S, using the stream in TOK.
357 Return NULL on failure, else S.
359 On entry, tok->decoding_buffer will be one of:
360 1) NULL: need to call tok->decoding_readline to get a new line
361 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
362 stored the result in tok->decoding_buffer
363 3) PyStringObject *: previous call to fp_readl did not have enough room
364 (in the s buffer) to copy entire contents of the line read
365 by tok->decoding_readline. tok->decoding_buffer has the overflow.
366 In this case, fp_readl is called in a loop (with an expanded buffer)
367 until the buffer ends with a '\n' (or until the end of the file is
368 reached): see tok_nextc and its calls to decoding_fgets.
372 fp_readl(char *s
, int size
, struct tok_state
*tok
)
374 #ifndef Py_USING_UNICODE
375 /* In a non-Unicode built, this should never be called. */
376 Py_FatalError("fp_readl should not be called in this build.");
377 return NULL
; /* Keep compiler happy (not reachable) */
379 PyObject
* utf8
= NULL
;
380 PyObject
* buf
= tok
->decoding_buffer
;
384 /* Ask for one less byte so we can terminate it */
389 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
391 return error_ret(tok
);
393 tok
->decoding_buffer
= NULL
;
394 if (PyString_CheckExact(buf
))
398 utf8
= PyUnicode_AsUTF8String(buf
);
401 return error_ret(tok
);
403 str
= PyString_AsString(utf8
);
404 utf8len
= PyString_GET_SIZE(utf8
);
405 if (utf8len
> size
) {
406 tok
->decoding_buffer
= PyString_FromStringAndSize(str
+size
, utf8len
-size
);
407 if (tok
->decoding_buffer
== NULL
) {
409 return error_ret(tok
);
413 memcpy(s
, str
, utf8len
);
417 return NULL
; /* EOF */
422 /* Set the readline function for TOK to a StreamReader's
423 readline function. The StreamReader is named ENC.
425 This function is called from check_bom and check_coding_spec.
427 ENC is usually identical to the future value of tok->encoding,
428 except for the (currently unsupported) case of UTF-16.
430 Return 1 on success, 0 on failure. */
433 fp_setreadl(struct tok_state
*tok
, const char* enc
)
435 PyObject
*reader
, *stream
, *readline
;
437 /* XXX: constify filename argument. */
438 stream
= PyFile_FromFile(tok
->fp
, (char*)tok
->filename
, "rb", NULL
);
442 reader
= PyCodec_StreamReader(enc
, stream
, NULL
);
447 readline
= PyObject_GetAttrString(reader
, "readline");
449 if (readline
== NULL
)
452 tok
->decoding_readline
= readline
;
456 /* Fetch the next byte from TOK. */
458 static int fp_getc(struct tok_state
*tok
) {
459 return getc(tok
->fp
);
462 /* Unfetch the last byte back into TOK. */
464 static void fp_ungetc(int c
, struct tok_state
*tok
) {
468 /* Read a line of input from TOK. Determine encoding
472 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
477 if (tok
->decoding_state
< 0) {
478 /* We already have a codec associated with
480 line
= fp_readl(s
, size
, tok
);
482 } else if (tok
->decoding_state
> 0) {
483 /* We want a 'raw' read. */
484 line
= Py_UniversalNewlineFgets(s
, size
,
488 /* We have not yet determined the encoding.
489 If an encoding is found, use the file-pointer
490 reader functions from now on. */
491 if (!check_bom(fp_getc
, fp_ungetc
, fp_setreadl
, tok
))
492 return error_ret(tok
);
493 assert(tok
->decoding_state
!= 0);
496 if (line
!= NULL
&& tok
->lineno
< 2 && !tok
->read_coding_spec
) {
497 if (!check_coding_spec(line
, strlen(line
), tok
, fp_setreadl
)) {
498 return error_ret(tok
);
502 /* The default encoding is ASCII, so make sure we don't have any
503 non-ASCII bytes in it. */
504 if (line
&& !tok
->encoding
) {
506 for (c
= (unsigned char *)line
; *c
; c
++)
514 /* Need to add 1 to the line number, since this line
515 has not been counted, yet. */
517 "Non-ASCII character '\\x%.2x' "
518 "in file %.200s on line %i, "
519 "but no encoding declared; "
520 "see http://www.python.org/peps/pep-0263.html for details",
521 badchar
, tok
->filename
, tok
->lineno
+ 1);
522 PyErr_SetString(PyExc_SyntaxError
, buf
);
523 return error_ret(tok
);
530 decoding_feof(struct tok_state
*tok
)
532 if (tok
->decoding_state
>= 0) {
533 return feof(tok
->fp
);
535 PyObject
* buf
= tok
->decoding_buffer
;
537 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
542 tok
->decoding_buffer
= buf
;
545 return PyObject_Length(buf
) == 0;
549 /* Fetch a byte from TOK, using the string buffer. */
552 buf_getc(struct tok_state
*tok
) {
553 return Py_CHARMASK(*tok
->str
++);
556 /* Unfetch a byte from TOK, using the string buffer. */
559 buf_ungetc(int c
, struct tok_state
*tok
) {
561 assert(Py_CHARMASK(*tok
->str
) == c
); /* tok->cur may point to read-only segment */
564 /* Set the readline function for TOK to ENC. For the string-based
565 tokenizer, this means to just record the encoding. */
568 buf_setreadl(struct tok_state
*tok
, const char* enc
) {
573 /* Return a UTF-8 encoding Python string object from the
574 C byte string STR, which is encoded with ENC. */
576 #ifdef Py_USING_UNICODE
578 translate_into_utf8(const char* str
, const char* enc
) {
580 PyObject
* buf
= PyUnicode_Decode(str
, strlen(str
), enc
, NULL
);
583 utf8
= PyUnicode_AsUTF8String(buf
);
589 /* Decode a byte string STR for use as the buffer of TOK.
590 Look for encoding declarations inside STR, and record them
594 decode_str(const char *str
, struct tok_state
*tok
)
596 PyObject
* utf8
= NULL
;
598 const char *newl
[2] = {NULL
, NULL
};
602 if (!check_bom(buf_getc
, buf_ungetc
, buf_setreadl
, tok
))
603 return error_ret(tok
);
604 str
= tok
->str
; /* string after BOM if any */
606 #ifdef Py_USING_UNICODE
607 if (tok
->enc
!= NULL
) {
608 utf8
= translate_into_utf8(str
, tok
->enc
);
610 return error_ret(tok
);
611 str
= PyString_AsString(utf8
);
614 for (s
= str
;; s
++) {
615 if (*s
== '\0') break;
616 else if (*s
== '\n') {
620 if (lineno
== 2) break;
624 /* need to check line 1 and 2 separately since check_coding_spec
625 assumes a single line as input */
627 if (!check_coding_spec(str
, newl
[0] - str
, tok
, buf_setreadl
))
628 return error_ret(tok
);
629 if (tok
->enc
== NULL
&& newl
[1]) {
630 if (!check_coding_spec(newl
[0]+1, newl
[1] - newl
[0],
632 return error_ret(tok
);
635 #ifdef Py_USING_UNICODE
636 if (tok
->enc
!= NULL
) {
637 assert(utf8
== NULL
);
638 utf8
= translate_into_utf8(str
, tok
->enc
);
640 return error_ret(tok
);
641 str
= PyString_AsString(utf8
);
644 assert(tok
->decoding_buffer
== NULL
);
645 tok
->decoding_buffer
= utf8
; /* CAUTION */
651 /* Set up tokenizer for string */
654 PyTokenizer_FromString(const char *str
)
656 struct tok_state
*tok
= tok_new();
659 str
= (char *)decode_str(str
, tok
);
661 PyTokenizer_Free(tok
);
665 /* XXX: constify members. */
666 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= (char*)str
;
671 /* Set up tokenizer for file */
674 PyTokenizer_FromFile(FILE *fp
, char *ps1
, char *ps2
)
676 struct tok_state
*tok
= tok_new();
679 if ((tok
->buf
= (char *)PyMem_MALLOC(BUFSIZ
)) == NULL
) {
680 PyTokenizer_Free(tok
);
683 tok
->cur
= tok
->inp
= tok
->buf
;
684 tok
->end
= tok
->buf
+ BUFSIZ
;
687 tok
->nextprompt
= ps2
;
692 /* Free a tok_state structure */
695 PyTokenizer_Free(struct tok_state
*tok
)
697 if (tok
->encoding
!= NULL
)
698 PyMem_FREE(tok
->encoding
);
700 Py_XDECREF(tok
->decoding_readline
);
701 Py_XDECREF(tok
->decoding_buffer
);
703 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
)
704 PyMem_FREE(tok
->buf
);
708 #if !defined(PGEN) && defined(Py_USING_UNICODE)
710 tok_stdin_decode(struct tok_state
*tok
, char **inp
)
712 PyObject
*enc
, *sysstdin
, *decoded
, *utf8
;
713 const char *encoding
;
716 if (PySys_GetFile((char *)"stdin", NULL
) != stdin
)
718 sysstdin
= PySys_GetObject("stdin");
719 if (sysstdin
== NULL
|| !PyFile_Check(sysstdin
))
722 enc
= ((PyFileObject
*)sysstdin
)->f_encoding
;
723 if (enc
== NULL
|| !PyString_Check(enc
))
727 encoding
= PyString_AsString(enc
);
728 decoded
= PyUnicode_Decode(*inp
, strlen(*inp
), encoding
, NULL
);
732 utf8
= PyUnicode_AsEncodedString(decoded
, "utf-8", NULL
);
737 assert(PyString_Check(utf8
));
738 converted
= new_string(PyString_AS_STRING(utf8
),
739 PyString_GET_SIZE(utf8
));
741 if (converted
== NULL
)
746 if (tok
->encoding
!= NULL
)
747 PyMem_FREE(tok
->encoding
);
748 tok
->encoding
= new_string(encoding
, strlen(encoding
));
749 if (tok
->encoding
== NULL
)
761 /* Fallback to iso-8859-1: for backward compatibility */
768 /* Get next char, updating state; error code goes into tok->done */
771 tok_nextc(register struct tok_state
*tok
)
774 if (tok
->cur
!= tok
->inp
) {
775 return Py_CHARMASK(*tok
->cur
++); /* Fast path */
777 if (tok
->done
!= E_OK
)
779 if (tok
->fp
== NULL
) {
780 char *end
= strchr(tok
->inp
, '\n');
784 end
= strchr(tok
->inp
, '\0');
785 if (end
== tok
->inp
) {
790 if (tok
->start
== NULL
)
792 tok
->line_start
= tok
->cur
;
795 return Py_CHARMASK(*tok
->cur
++);
797 if (tok
->prompt
!= NULL
) {
798 char *newtok
= PyOS_Readline(stdin
, stdout
, tok
->prompt
);
799 if (tok
->nextprompt
!= NULL
)
800 tok
->prompt
= tok
->nextprompt
;
803 else if (*newtok
== '\0') {
807 #if !defined(PGEN) && defined(Py_USING_UNICODE)
808 else if (tok_stdin_decode(tok
, &newtok
) != 0)
811 else if (tok
->start
!= NULL
) {
812 size_t start
= tok
->start
- tok
->buf
;
813 size_t oldlen
= tok
->cur
- tok
->buf
;
814 size_t newlen
= oldlen
+ strlen(newtok
);
815 char *buf
= tok
->buf
;
816 buf
= (char *)PyMem_REALLOC(buf
, newlen
+1);
819 PyMem_FREE(tok
->buf
);
826 tok
->cur
= tok
->buf
+ oldlen
;
827 tok
->line_start
= tok
->cur
;
828 strcpy(tok
->buf
+ oldlen
, newtok
);
830 tok
->inp
= tok
->buf
+ newlen
;
831 tok
->end
= tok
->inp
+ 1;
832 tok
->start
= tok
->buf
+ start
;
836 if (tok
->buf
!= NULL
)
837 PyMem_FREE(tok
->buf
);
839 tok
->line_start
= tok
->buf
;
841 tok
->line_start
= tok
->buf
;
842 tok
->inp
= strchr(tok
->buf
, '\0');
843 tok
->end
= tok
->inp
+ 1;
850 if (tok
->start
== NULL
) {
851 if (tok
->buf
== NULL
) {
853 PyMem_MALLOC(BUFSIZ
);
854 if (tok
->buf
== NULL
) {
858 tok
->end
= tok
->buf
+ BUFSIZ
;
860 if (decoding_fgets(tok
->buf
, (int)(tok
->end
- tok
->buf
),
867 tok
->inp
= strchr(tok
->buf
, '\0');
868 done
= tok
->inp
[-1] == '\n';
872 cur
= tok
->cur
- tok
->buf
;
873 if (decoding_feof(tok
)) {
881 /* Read until '\n' or EOF */
883 Py_ssize_t curstart
= tok
->start
== NULL
? -1 :
884 tok
->start
- tok
->buf
;
885 Py_ssize_t curvalid
= tok
->inp
- tok
->buf
;
886 Py_ssize_t newsize
= curvalid
+ BUFSIZ
;
887 char *newbuf
= tok
->buf
;
888 newbuf
= (char *)PyMem_REALLOC(newbuf
,
890 if (newbuf
== NULL
) {
896 tok
->inp
= tok
->buf
+ curvalid
;
897 tok
->end
= tok
->buf
+ newsize
;
898 tok
->start
= curstart
< 0 ? NULL
:
900 if (decoding_fgets(tok
->inp
,
901 (int)(tok
->end
- tok
->inp
),
903 /* Break out early on decoding
904 errors, as tok->buf will be NULL
906 if (tok
->decoding_erred
)
908 /* Last line does not end in \n,
910 strcpy(tok
->inp
, "\n");
912 tok
->inp
= strchr(tok
->inp
, '\0');
913 done
= tok
->inp
[-1] == '\n';
915 if (tok
->buf
!= NULL
) {
916 tok
->cur
= tok
->buf
+ cur
;
917 tok
->line_start
= tok
->cur
;
918 /* replace "\r\n" with "\n" */
919 /* For Mac leave the \r, giving a syntax error */
921 if (pt
>= tok
->buf
&& *pt
== '\r') {
928 if (tok
->done
!= E_OK
) {
929 if (tok
->prompt
!= NULL
)
930 PySys_WriteStderr("\n");
939 /* Back-up one character */
942 tok_backup(register struct tok_state
*tok
, register int c
)
945 if (--tok
->cur
< tok
->buf
)
946 Py_FatalError("tok_backup: begin of buffer");
953 /* Return the token corresponding to a single character */
956 PyToken_OneChar(int c
)
959 case '(': return LPAR
;
960 case ')': return RPAR
;
961 case '[': return LSQB
;
962 case ']': return RSQB
;
963 case ':': return COLON
;
964 case ',': return COMMA
;
965 case ';': return SEMI
;
966 case '+': return PLUS
;
967 case '-': return MINUS
;
968 case '*': return STAR
;
969 case '/': return SLASH
;
970 case '|': return VBAR
;
971 case '&': return AMPER
;
972 case '<': return LESS
;
973 case '>': return GREATER
;
974 case '=': return EQUAL
;
975 case '.': return DOT
;
976 case '%': return PERCENT
;
977 case '`': return BACKQUOTE
;
978 case '{': return LBRACE
;
979 case '}': return RBRACE
;
980 case '^': return CIRCUMFLEX
;
981 case '~': return TILDE
;
989 PyToken_TwoChars(int c1
, int c2
)
994 case '=': return EQEQUAL
;
999 case '=': return NOTEQUAL
;
1004 case '>': return NOTEQUAL
;
1005 case '=': return LESSEQUAL
;
1006 case '<': return LEFTSHIFT
;
1011 case '=': return GREATEREQUAL
;
1012 case '>': return RIGHTSHIFT
;
1017 case '=': return PLUSEQUAL
;
1022 case '=': return MINEQUAL
;
1027 case '*': return DOUBLESTAR
;
1028 case '=': return STAREQUAL
;
1033 case '/': return DOUBLESLASH
;
1034 case '=': return SLASHEQUAL
;
1039 case '=': return VBAREQUAL
;
1044 case '=': return PERCENTEQUAL
;
1049 case '=': return AMPEREQUAL
;
1054 case '=': return CIRCUMFLEXEQUAL
;
1062 PyToken_ThreeChars(int c1
, int c2
, int c3
)
1070 return LEFTSHIFTEQUAL
;
1080 return RIGHTSHIFTEQUAL
;
1090 return DOUBLESTAREQUAL
;
1100 return DOUBLESLASHEQUAL
;
1110 indenterror(struct tok_state
*tok
)
1112 if (tok
->alterror
) {
1113 tok
->done
= E_TABSPACE
;
1114 tok
->cur
= tok
->inp
;
1117 if (tok
->altwarning
) {
1118 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
1119 "in indentation\n", tok
->filename
);
1120 tok
->altwarning
= 0;
1126 /* Get next token, after space stripping etc. */
1129 tok_get(register struct tok_state
*tok
, char **p_start
, char **p_end
)
1134 *p_start
= *p_end
= NULL
;
1139 /* Get indentation level */
1141 register int col
= 0;
1142 register int altcol
= 0;
1148 else if (c
== '\t') {
1149 col
= (col
/tok
->tabsize
+ 1) * tok
->tabsize
;
1150 altcol
= (altcol
/tok
->alttabsize
+ 1)
1153 else if (c
== '\014') /* Control-L (formfeed) */
1154 col
= altcol
= 0; /* For Emacs users */
1159 if (c
== '#' || c
== '\n') {
1160 /* Lines with only whitespace and/or comments
1161 shouldn't affect the indentation and are
1162 not passed to the parser as NEWLINE tokens,
1163 except *totally* empty lines in interactive
1164 mode, which signal the end of a command group. */
1165 if (col
== 0 && c
== '\n' && tok
->prompt
!= NULL
)
1166 blankline
= 0; /* Let it through */
1168 blankline
= 1; /* Ignore completely */
1169 /* We can't jump back right here since we still
1170 may need to skip to the end of a comment */
1172 if (!blankline
&& tok
->level
== 0) {
1173 if (col
== tok
->indstack
[tok
->indent
]) {
1175 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1176 if (indenterror(tok
))
1180 else if (col
> tok
->indstack
[tok
->indent
]) {
1181 /* Indent -- always one */
1182 if (tok
->indent
+1 >= MAXINDENT
) {
1183 tok
->done
= E_TOODEEP
;
1184 tok
->cur
= tok
->inp
;
1187 if (altcol
<= tok
->altindstack
[tok
->indent
]) {
1188 if (indenterror(tok
))
1192 tok
->indstack
[++tok
->indent
] = col
;
1193 tok
->altindstack
[tok
->indent
] = altcol
;
1195 else /* col < tok->indstack[tok->indent] */ {
1196 /* Dedent -- any number, must be consistent */
1197 while (tok
->indent
> 0 &&
1198 col
< tok
->indstack
[tok
->indent
]) {
1202 if (col
!= tok
->indstack
[tok
->indent
]) {
1203 tok
->done
= E_DEDENT
;
1204 tok
->cur
= tok
->inp
;
1207 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1208 if (indenterror(tok
))
1215 tok
->start
= tok
->cur
;
1217 /* Return pending indents/dedents */
1218 if (tok
->pendin
!= 0) {
1219 if (tok
->pendin
< 0) {
1234 } while (c
== ' ' || c
== '\t' || c
== '\014');
1236 /* Set start of current token */
1237 tok
->start
= tok
->cur
- 1;
1239 /* Skip comment, while looking for tab-setting magic */
1241 static char *tabforms
[] = {
1242 "tab-width:", /* Emacs */
1243 ":tabstop=", /* vim, full form */
1244 ":ts=", /* vim, abbreviated form */
1245 "set tabsize=", /* will vi never die? */
1246 /* more templates can be added here to support other editors */
1252 *tp
++ = c
= tok_nextc(tok
);
1253 } while (c
!= EOF
&& c
!= '\n' &&
1254 (size_t)(tp
- cbuf
+ 1) < sizeof(cbuf
));
1257 cp
< tabforms
+ sizeof(tabforms
)/sizeof(tabforms
[0]);
1259 if ((tp
= strstr(cbuf
, *cp
))) {
1260 int newsize
= atoi(tp
+ strlen(*cp
));
1262 if (newsize
>= 1 && newsize
<= 40) {
1263 tok
->tabsize
= newsize
;
1266 "Tab size set to %d\n",
1271 while (c
!= EOF
&& c
!= '\n')
1275 /* Check for EOF and errors now */
1277 return tok
->done
== E_EOF
? ENDMARKER
: ERRORTOKEN
;
1280 /* Identifier (most frequent token!) */
1281 if (isalpha(c
) || c
== '_') {
1282 /* Process r"", u"" and ur"" */
1287 if (c
== 'r' || c
== 'R')
1289 if (c
== '"' || c
== '\'')
1295 if (c
== '"' || c
== '\'')
1301 if (c
== 'r' || c
== 'R')
1303 if (c
== '"' || c
== '\'')
1307 while (isalnum(c
) || c
== '_') {
1311 *p_start
= tok
->start
;
1319 if (blankline
|| tok
->level
> 0)
1321 *p_start
= tok
->start
;
1322 *p_end
= tok
->cur
- 1; /* Leave '\n' out of the string */
1327 /* Period or number starting with period? */
1335 *p_start
= tok
->start
;
1344 /* Hex, octal or binary -- maybe. */
1348 #ifndef WITHOUT_COMPLEX
1349 if (c
== 'j' || c
== 'J')
1352 if (c
== 'x' || c
== 'X') {
1357 tok
->done
= E_TOKEN
;
1363 } while (isxdigit(c
));
1365 else if (c
== 'o' || c
== 'O') {
1368 if (c
< '0' || c
>= '8') {
1369 tok
->done
= E_TOKEN
;
1375 } while ('0' <= c
&& c
< '8');
1377 else if (c
== 'b' || c
== 'B') {
1380 if (c
!= '0' && c
!= '1') {
1381 tok
->done
= E_TOKEN
;
1387 } while (c
== '0' || c
== '1');
1390 int found_decimal
= 0;
1391 /* Octal; c is first char of it */
1392 /* There's no 'isoctdigit' macro, sigh */
1393 while ('0' <= c
&& c
< '8') {
1400 } while (isdigit(c
));
1404 else if (c
== 'e' || c
== 'E')
1406 #ifndef WITHOUT_COMPLEX
1407 else if (c
== 'j' || c
== 'J')
1410 else if (found_decimal
) {
1411 tok
->done
= E_TOKEN
;
1416 if (c
== 'l' || c
== 'L')
1423 } while (isdigit(c
));
1424 if (c
== 'l' || c
== 'L')
1427 /* Accept floating point numbers. */
1433 } while (isdigit(c
));
1435 if (c
== 'e' || c
== 'E') {
1439 if (c
== '+' || c
== '-')
1442 tok
->done
= E_TOKEN
;
1448 } while (isdigit(c
));
1450 #ifndef WITHOUT_COMPLEX
1451 if (c
== 'j' || c
== 'J')
1452 /* Imaginary part */
1459 *p_start
= tok
->start
;
1466 if (c
== '\'' || c
== '"') {
1467 Py_ssize_t quote2
= tok
->cur
- tok
->start
+ 1;
1480 tok
->cont_line
= 1; /* multiline string. */
1482 else if (c
== EOF
) {
1487 tok
->cur
= tok
->inp
;
1490 else if (c
== quote
) {
1492 if (tok
->cur
- tok
->start
== quote2
) {
1501 if (!triple
|| tripcount
== 3)
1504 else if (c
== '\\') {
1509 tok
->cur
= tok
->inp
;
1516 *p_start
= tok
->start
;
1521 /* Line continuation */
1525 tok
->done
= E_LINECONT
;
1526 tok
->cur
= tok
->inp
;
1530 goto again
; /* Read next line */
1533 /* Check for two-character token */
1535 int c2
= tok_nextc(tok
);
1536 int token
= PyToken_TwoChars(c
, c2
);
1538 if (Py_Py3kWarningFlag
&& token
== NOTEQUAL
&& c
== '<') {
1539 if (PyErr_WarnExplicit(PyExc_DeprecationWarning
,
1540 "<> not supported in 3.x; use !=",
1541 tok
->filename
, tok
->lineno
,
1548 int c3
= tok_nextc(tok
);
1549 int token3
= PyToken_ThreeChars(c
, c2
, c3
);
1553 tok_backup(tok
, c3
);
1555 *p_start
= tok
->start
;
1559 tok_backup(tok
, c2
);
1562 /* Keep track of parentheses nesting level */
1576 /* Punctuation character */
1577 *p_start
= tok
->start
;
1579 return PyToken_OneChar(c
);
1583 PyTokenizer_Get(struct tok_state
*tok
, char **p_start
, char **p_end
)
1585 int result
= tok_get(tok
, p_start
, p_end
);
1586 if (tok
->decoding_erred
) {
1587 result
= ERRORTOKEN
;
1588 tok
->done
= E_DECODE
;
1593 /* This function is only called from parsetok. However, it cannot live
1594 there, as it must be empty for PGEN, and we can check for PGEN only
1597 #if defined(PGEN) || !defined(Py_USING_UNICODE)
1599 PyTokenizer_RestoreEncoding(struct tok_state
* tok
, int len
, int* offset
)
1604 #ifdef Py_USING_UNICODE
1606 dec_utf8(const char *enc
, const char *text
, size_t len
) {
1607 PyObject
*ret
= NULL
;
1608 PyObject
*unicode_text
= PyUnicode_DecodeUTF8(text
, len
, "replace");
1610 ret
= PyUnicode_AsEncodedString(unicode_text
, enc
, "replace");
1611 Py_DECREF(unicode_text
);
1619 PyTokenizer_RestoreEncoding(struct tok_state
* tok
, int len
, int *offset
)
1622 if (tok
->encoding
) {
1623 /* convert source to original encondig */
1624 PyObject
*lineobj
= dec_utf8(tok
->encoding
, tok
->buf
, len
);
1625 if (lineobj
!= NULL
) {
1626 int linelen
= PyString_Size(lineobj
);
1627 const char *line
= PyString_AsString(lineobj
);
1628 text
= PyObject_MALLOC(linelen
+ 1);
1629 if (text
!= NULL
&& line
!= NULL
) {
1631 strncpy(text
, line
, linelen
);
1632 text
[linelen
] = '\0';
1636 /* adjust error offset */
1638 PyObject
*offsetobj
= dec_utf8(tok
->encoding
,
1639 tok
->buf
, *offset
-1);
1641 *offset
= PyString_Size(offsetobj
) + 1;
1642 Py_DECREF(offsetobj
);
1651 #endif /* defined(Py_USING_UNICODE) */
1658 tok_dump(int type
, char *start
, char *end
)
1660 printf("%s", _PyParser_TokenNames
[type
]);
1661 if (type
== NAME
|| type
== NUMBER
|| type
== STRING
|| type
== OP
)
1662 printf("(%.*s)", (int)(end
- start
), start
);