2 /* Tokenizer implementation */
5 #include "pgenheaders.h"
10 #include "tokenizer.h"
14 #include "unicodeobject.h"
15 #include "stringobject.h"
16 #include "fileobject.h"
21 extern char *PyOS_Readline(FILE *, FILE *, char *);
22 /* Return malloc'ed string including trailing \n;
23 empty malloc'ed string for EOF;
24 NULL if interrupted */
26 /* Don't ever change this -- it would break the portability of Python code */
29 /* Convert a possibly signed character to a nonnegative int */
30 /* XXX This assumes characters are 8 bits wide */
31 #ifdef __CHAR_UNSIGNED__
32 #define Py_CHARMASK(c) (c)
34 #define Py_CHARMASK(c) ((c) & 0xff)
38 static struct tok_state
*tok_new(void);
39 static int tok_nextc(struct tok_state
*tok
);
40 static void tok_backup(struct tok_state
*tok
, int c
);
44 char *_PyParser_TokenNames
[] = {
96 /* This table must match the #defines in token.h! */
103 /* Create and initialize a new tok_state structure */
105 static struct tok_state
*
108 struct tok_state
*tok
= PyMem_NEW(struct tok_state
, 1);
111 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= tok
->start
= NULL
;
114 tok
->tabsize
= TABSIZE
;
116 tok
->indstack
[0] = 0;
119 tok
->prompt
= tok
->nextprompt
= NULL
;
122 tok
->filename
= NULL
;
126 tok
->altindstack
[0] = 0;
127 tok
->decoding_state
= 0;
128 tok
->decoding_erred
= 0;
129 tok
->read_coding_spec
= 0;
130 tok
->encoding
= NULL
;
133 tok
->decoding_readline
= NULL
;
134 tok
->decoding_buffer
= NULL
;
142 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
144 return fgets(s
, size
, tok
->fp
);
148 decoding_feof(struct tok_state
*tok
)
150 return feof(tok
->fp
);
154 decode_str(const char *str
, struct tok_state
*tok
)
162 error_ret(struct tok_state
*tok
) /* XXX */
164 tok
->decoding_erred
= 1;
165 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
) /* see PyTokenizer_Free */
166 PyObject_FREE(tok
->buf
);
168 return NULL
; /* as if it were EOF */
172 new_string(const char *s
, Py_ssize_t len
)
174 char* result
= (char *)PyObject_MALLOC(len
+ 1);
175 if (result
!= NULL
) {
176 memcpy(result
, s
, len
);
183 get_normal_name(char *s
) /* for utf-8 and latin-1 */
187 for (i
= 0; i
< 12; i
++) {
189 if (c
== '\0') break;
190 else if (c
== '_') buf
[i
] = '-';
191 else buf
[i
] = tolower(c
);
194 if (strcmp(buf
, "utf-8") == 0 ||
195 strncmp(buf
, "utf-8-", 6) == 0) return "utf-8";
196 else if (strcmp(buf
, "latin-1") == 0 ||
197 strcmp(buf
, "iso-8859-1") == 0 ||
198 strcmp(buf
, "iso-latin-1") == 0 ||
199 strncmp(buf
, "latin-1-", 8) == 0 ||
200 strncmp(buf
, "iso-8859-1-", 11) == 0 ||
201 strncmp(buf
, "iso-latin-1-", 12) == 0) return "iso-8859-1";
205 /* Return the coding spec in S, or NULL if none is found. */
208 get_coding_spec(const char *s
, Py_ssize_t size
)
211 /* Coding spec must be in a comment, and that comment must be
212 * the only statement on the source code line. */
213 for (i
= 0; i
< size
- 6; i
++) {
216 if (s
[i
] != ' ' && s
[i
] != '\t' && s
[i
] != '\014')
219 for (; i
< size
- 6; i
++) { /* XXX inefficient search */
220 const char* t
= s
+ i
;
221 if (strncmp(t
, "coding", 6) == 0) {
222 const char* begin
= NULL
;
224 if (t
[0] != ':' && t
[0] != '=')
228 } while (t
[0] == '\x20' || t
[0] == '\t');
231 while (isalnum(Py_CHARMASK(t
[0])) ||
232 t
[0] == '-' || t
[0] == '_' || t
[0] == '.')
236 char* r
= new_string(begin
, t
- begin
);
237 char* q
= get_normal_name(r
);
240 r
= new_string(q
, strlen(q
));
249 /* Check whether the line contains a coding spec. If it does,
250 invoke the set_readline function for the new encoding.
251 This function receives the tok_state and the new encoding.
252 Return 1 on success, 0 on failure. */
255 check_coding_spec(const char* line
, Py_ssize_t size
, struct tok_state
*tok
,
256 int set_readline(struct tok_state
*, const char *))
262 /* It's a continuation line, so it can't be a coding spec. */
264 cs
= get_coding_spec(line
, size
);
266 tok
->read_coding_spec
= 1;
267 if (tok
->encoding
== NULL
) {
268 assert(tok
->decoding_state
== 1); /* raw */
269 if (strcmp(cs
, "utf-8") == 0 ||
270 strcmp(cs
, "iso-8859-1") == 0) {
273 #ifdef Py_USING_UNICODE
274 r
= set_readline(tok
, cs
);
277 tok
->decoding_state
= -1;
282 /* Without Unicode support, we cannot
283 process the coding spec. Since there
284 won't be any Unicode literals, that
289 } else { /* then, compare cs with BOM */
290 r
= (strcmp(tok
->encoding
, cs
) == 0);
298 PyErr_Format(PyExc_SyntaxError
, "encoding problem: %s", cs
);
303 /* See whether the file starts with a BOM. If it does,
304 invoke the set_readline function with the new encoding.
305 Return 1 on success, 0 on failure. */
308 check_bom(int get_char(struct tok_state
*),
309 void unget_char(int, struct tok_state
*),
310 int set_readline(struct tok_state
*, const char *),
311 struct tok_state
*tok
)
313 int ch
= get_char(tok
);
314 tok
->decoding_state
= 1;
317 } else if (ch
== 0xEF) {
318 ch
= get_char(tok
); if (ch
!= 0xBB) goto NON_BOM
;
319 ch
= get_char(tok
); if (ch
!= 0xBF) goto NON_BOM
;
321 /* Disable support for UTF-16 BOMs until a decision
322 is made whether this needs to be supported. */
323 } else if (ch
== 0xFE) {
324 ch
= get_char(tok
); if (ch
!= 0xFF) goto NON_BOM
;
325 if (!set_readline(tok
, "utf-16-be")) return 0;
326 tok
->decoding_state
= -1;
327 } else if (ch
== 0xFF) {
328 ch
= get_char(tok
); if (ch
!= 0xFE) goto NON_BOM
;
329 if (!set_readline(tok
, "utf-16-le")) return 0;
330 tok
->decoding_state
= -1;
336 if (tok
->encoding
!= NULL
)
337 PyObject_FREE(tok
->encoding
);
338 tok
->encoding
= new_string("utf-8", 5); /* resulting is in utf-8 */
341 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
342 unget_char(0xFF, tok
); /* XXX this will cause a syntax error */
346 /* Read a line of text from TOK into S, using the stream in TOK.
347 Return NULL on failure, else S.
349 On entry, tok->decoding_buffer will be one of:
350 1) NULL: need to call tok->decoding_readline to get a new line
351 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
352 stored the result in tok->decoding_buffer
353 3) PyStringObject *: previous call to fp_readl did not have enough room
354 (in the s buffer) to copy entire contents of the line read
355 by tok->decoding_readline. tok->decoding_buffer has the overflow.
356 In this case, fp_readl is called in a loop (with an expanded buffer)
357 until the buffer ends with a '\n' (or until the end of the file is
358 reached): see tok_nextc and its calls to decoding_fgets.
362 fp_readl(char *s
, int size
, struct tok_state
*tok
)
364 #ifndef Py_USING_UNICODE
365 /* In a non-Unicode built, this should never be called. */
366 Py_FatalError("fp_readl should not be called in this build.");
367 return NULL
; /* Keep compiler happy (not reachable) */
369 PyObject
* utf8
= NULL
;
370 PyObject
* buf
= tok
->decoding_buffer
;
374 /* Ask for one less byte so we can terminate it */
379 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
381 return error_ret(tok
);
383 tok
->decoding_buffer
= NULL
;
384 if (PyString_CheckExact(buf
))
388 utf8
= PyUnicode_AsUTF8String(buf
);
391 return error_ret(tok
);
393 str
= PyString_AsString(utf8
);
394 utf8len
= PyString_GET_SIZE(utf8
);
395 if (utf8len
> size
) {
396 tok
->decoding_buffer
= PyString_FromStringAndSize(str
+size
, utf8len
-size
);
397 if (tok
->decoding_buffer
== NULL
) {
399 return error_ret(tok
);
403 memcpy(s
, str
, utf8len
);
406 if (utf8len
== 0) return NULL
; /* EOF */
411 /* Set the readline function for TOK to a StreamReader's
412 readline function. The StreamReader is named ENC.
414 This function is called from check_bom and check_coding_spec.
416 ENC is usually identical to the future value of tok->encoding,
417 except for the (currently unsupported) case of UTF-16.
419 Return 1 on success, 0 on failure. */
422 fp_setreadl(struct tok_state
*tok
, const char* enc
)
424 PyObject
*reader
, *stream
, *readline
;
426 /* XXX: constify filename argument. */
427 stream
= PyFile_FromFile(tok
->fp
, (char*)tok
->filename
, "rb", NULL
);
431 reader
= PyCodec_StreamReader(enc
, stream
, NULL
);
436 readline
= PyObject_GetAttrString(reader
, "readline");
438 if (readline
== NULL
)
441 tok
->decoding_readline
= readline
;
445 /* Fetch the next byte from TOK. */
447 static int fp_getc(struct tok_state
*tok
) {
448 return getc(tok
->fp
);
451 /* Unfetch the last byte back into TOK. */
453 static void fp_ungetc(int c
, struct tok_state
*tok
) {
457 /* Read a line of input from TOK. Determine encoding
461 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
466 if (tok
->decoding_state
< 0) {
467 /* We already have a codec associated with
469 line
= fp_readl(s
, size
, tok
);
471 } else if (tok
->decoding_state
> 0) {
472 /* We want a 'raw' read. */
473 line
= Py_UniversalNewlineFgets(s
, size
,
477 /* We have not yet determined the encoding.
478 If an encoding is found, use the file-pointer
479 reader functions from now on. */
480 if (!check_bom(fp_getc
, fp_ungetc
, fp_setreadl
, tok
))
481 return error_ret(tok
);
482 assert(tok
->decoding_state
!= 0);
485 if (line
!= NULL
&& tok
->lineno
< 2 && !tok
->read_coding_spec
) {
486 if (!check_coding_spec(line
, strlen(line
), tok
, fp_setreadl
)) {
487 return error_ret(tok
);
491 /* The default encoding is ASCII, so make sure we don't have any
492 non-ASCII bytes in it. */
493 if (line
&& !tok
->encoding
) {
495 for (c
= (unsigned char *)line
; *c
; c
++)
503 /* Need to add 1 to the line number, since this line
504 has not been counted, yet. */
506 "Non-ASCII character '\\x%.2x' "
507 "in file %.200s on line %i, "
508 "but no encoding declared; "
509 "see http://www.python.org/peps/pep-0263.html for details",
510 badchar
, tok
->filename
, tok
->lineno
+ 1);
511 PyErr_SetString(PyExc_SyntaxError
, buf
);
512 return error_ret(tok
);
519 decoding_feof(struct tok_state
*tok
)
521 if (tok
->decoding_state
>= 0) {
522 return feof(tok
->fp
);
524 PyObject
* buf
= tok
->decoding_buffer
;
526 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
531 tok
->decoding_buffer
= buf
;
534 return PyObject_Length(buf
) == 0;
538 /* Fetch a byte from TOK, using the string buffer. */
541 buf_getc(struct tok_state
*tok
) {
542 return Py_CHARMASK(*tok
->str
++);
545 /* Unfetch a byte from TOK, using the string buffer. */
548 buf_ungetc(int c
, struct tok_state
*tok
) {
550 assert(Py_CHARMASK(*tok
->str
) == c
); /* tok->cur may point to read-only segment */
553 /* Set the readline function for TOK to ENC. For the string-based
554 tokenizer, this means to just record the encoding. */
557 buf_setreadl(struct tok_state
*tok
, const char* enc
) {
562 /* Return a UTF-8 encoding Python string object from the
563 C byte string STR, which is encoded with ENC. */
565 #ifdef Py_USING_UNICODE
567 translate_into_utf8(const char* str
, const char* enc
) {
569 PyObject
* buf
= PyUnicode_Decode(str
, strlen(str
), enc
, NULL
);
572 utf8
= PyUnicode_AsUTF8String(buf
);
578 /* Decode a byte string STR for use as the buffer of TOK.
579 Look for encoding declarations inside STR, and record them
583 decode_str(const char *str
, struct tok_state
*tok
)
585 PyObject
* utf8
= NULL
;
590 if (!check_bom(buf_getc
, buf_ungetc
, buf_setreadl
, tok
))
591 return error_ret(tok
);
592 str
= tok
->str
; /* string after BOM if any */
594 #ifdef Py_USING_UNICODE
595 if (tok
->enc
!= NULL
) {
596 utf8
= translate_into_utf8(str
, tok
->enc
);
598 return error_ret(tok
);
599 str
= PyString_AsString(utf8
);
602 for (s
= str
;; s
++) {
603 if (*s
== '\0') break;
604 else if (*s
== '\n') {
606 if (lineno
== 2) break;
610 if (!check_coding_spec(str
, s
- str
, tok
, buf_setreadl
))
611 return error_ret(tok
);
612 #ifdef Py_USING_UNICODE
613 if (tok
->enc
!= NULL
) {
614 assert(utf8
== NULL
);
615 utf8
= translate_into_utf8(str
, tok
->enc
);
617 PyErr_Format(PyExc_SyntaxError
,
618 "unknown encoding: %s", tok
->enc
);
619 return error_ret(tok
);
621 str
= PyString_AsString(utf8
);
624 assert(tok
->decoding_buffer
== NULL
);
625 tok
->decoding_buffer
= utf8
; /* CAUTION */
631 /* Set up tokenizer for string */
634 PyTokenizer_FromString(const char *str
)
636 struct tok_state
*tok
= tok_new();
639 str
= (char *)decode_str(str
, tok
);
641 PyTokenizer_Free(tok
);
645 /* XXX: constify members. */
646 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= (char*)str
;
651 /* Set up tokenizer for file */
654 PyTokenizer_FromFile(FILE *fp
, char *ps1
, char *ps2
)
656 struct tok_state
*tok
= tok_new();
659 if ((tok
->buf
= (char *)PyObject_MALLOC(BUFSIZ
)) == NULL
) {
660 PyTokenizer_Free(tok
);
663 tok
->cur
= tok
->inp
= tok
->buf
;
664 tok
->end
= tok
->buf
+ BUFSIZ
;
667 tok
->nextprompt
= ps2
;
672 /* Free a tok_state structure */
675 PyTokenizer_Free(struct tok_state
*tok
)
677 if (tok
->encoding
!= NULL
)
678 PyObject_FREE(tok
->encoding
);
680 Py_XDECREF(tok
->decoding_readline
);
681 Py_XDECREF(tok
->decoding_buffer
);
683 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
)
684 PyObject_FREE(tok
->buf
);
688 #if !defined(PGEN) && defined(Py_USING_UNICODE)
690 tok_stdin_decode(struct tok_state
*tok
, char **inp
)
692 PyObject
*enc
, *sysstdin
, *decoded
, *utf8
;
693 const char *encoding
;
696 if (PySys_GetFile((char *)"stdin", NULL
) != stdin
)
698 sysstdin
= PySys_GetObject("stdin");
699 if (sysstdin
== NULL
|| !PyFile_Check(sysstdin
))
702 enc
= ((PyFileObject
*)sysstdin
)->f_encoding
;
703 if (enc
== NULL
|| !PyString_Check(enc
))
707 encoding
= PyString_AsString(enc
);
708 decoded
= PyUnicode_Decode(*inp
, strlen(*inp
), encoding
, NULL
);
712 utf8
= PyUnicode_AsEncodedString(decoded
, "utf-8", NULL
);
717 assert(PyString_Check(utf8
));
718 converted
= new_string(PyString_AS_STRING(utf8
),
719 PyString_GET_SIZE(utf8
));
721 if (converted
== NULL
)
726 if (tok
->encoding
!= NULL
)
727 PyObject_FREE(tok
->encoding
);
728 tok
->encoding
= new_string(encoding
, strlen(encoding
));
729 if (tok
->encoding
== NULL
)
741 /* Fallback to iso-8859-1: for backward compatibility */
748 /* Get next char, updating state; error code goes into tok->done */
751 tok_nextc(register struct tok_state
*tok
)
754 if (tok
->cur
!= tok
->inp
) {
755 return Py_CHARMASK(*tok
->cur
++); /* Fast path */
757 if (tok
->done
!= E_OK
)
759 if (tok
->fp
== NULL
) {
760 char *end
= strchr(tok
->inp
, '\n');
764 end
= strchr(tok
->inp
, '\0');
765 if (end
== tok
->inp
) {
770 if (tok
->start
== NULL
)
772 tok
->line_start
= tok
->cur
;
775 return Py_CHARMASK(*tok
->cur
++);
777 if (tok
->prompt
!= NULL
) {
778 char *new = PyOS_Readline(stdin
, stdout
, tok
->prompt
);
779 if (tok
->nextprompt
!= NULL
)
780 tok
->prompt
= tok
->nextprompt
;
783 else if (*new == '\0') {
787 #if !defined(PGEN) && defined(Py_USING_UNICODE)
788 else if (tok_stdin_decode(tok
, &new) != 0)
791 else if (tok
->start
!= NULL
) {
792 size_t start
= tok
->start
- tok
->buf
;
793 size_t oldlen
= tok
->cur
- tok
->buf
;
794 size_t newlen
= oldlen
+ strlen(new);
795 char *buf
= tok
->buf
;
796 buf
= (char *)PyObject_REALLOC(buf
, newlen
+1);
799 PyObject_FREE(tok
->buf
);
806 tok
->cur
= tok
->buf
+ oldlen
;
807 tok
->line_start
= tok
->cur
;
808 strcpy(tok
->buf
+ oldlen
, new);
810 tok
->inp
= tok
->buf
+ newlen
;
811 tok
->end
= tok
->inp
+ 1;
812 tok
->start
= tok
->buf
+ start
;
816 if (tok
->buf
!= NULL
)
817 PyObject_FREE(tok
->buf
);
819 tok
->line_start
= tok
->buf
;
821 tok
->line_start
= tok
->buf
;
822 tok
->inp
= strchr(tok
->buf
, '\0');
823 tok
->end
= tok
->inp
+ 1;
830 if (tok
->start
== NULL
) {
831 if (tok
->buf
== NULL
) {
833 PyObject_MALLOC(BUFSIZ
);
834 if (tok
->buf
== NULL
) {
838 tok
->end
= tok
->buf
+ BUFSIZ
;
840 if (decoding_fgets(tok
->buf
, (int)(tok
->end
- tok
->buf
),
847 tok
->inp
= strchr(tok
->buf
, '\0');
848 done
= tok
->inp
[-1] == '\n';
852 cur
= tok
->cur
- tok
->buf
;
853 if (decoding_feof(tok
)) {
861 /* Read until '\n' or EOF */
863 Py_ssize_t curstart
= tok
->start
== NULL
? -1 :
864 tok
->start
- tok
->buf
;
865 Py_ssize_t curvalid
= tok
->inp
- tok
->buf
;
866 Py_ssize_t newsize
= curvalid
+ BUFSIZ
;
867 char *newbuf
= tok
->buf
;
868 newbuf
= (char *)PyObject_REALLOC(newbuf
,
870 if (newbuf
== NULL
) {
876 tok
->inp
= tok
->buf
+ curvalid
;
877 tok
->end
= tok
->buf
+ newsize
;
878 tok
->start
= curstart
< 0 ? NULL
:
880 if (decoding_fgets(tok
->inp
,
881 (int)(tok
->end
- tok
->inp
),
883 /* Break out early on decoding
884 errors, as tok->buf will be NULL
886 if (tok
->decoding_erred
)
888 /* Last line does not end in \n,
890 strcpy(tok
->inp
, "\n");
892 tok
->inp
= strchr(tok
->inp
, '\0');
893 done
= tok
->inp
[-1] == '\n';
895 tok
->cur
= tok
->buf
+ cur
;
896 tok
->line_start
= tok
->cur
;
897 /* replace "\r\n" with "\n" */
898 /* For Mac we leave the \r, giving a syntax error */
900 if (pt
>= tok
->buf
&& *pt
== '\r') {
906 if (tok
->done
!= E_OK
) {
907 if (tok
->prompt
!= NULL
)
908 PySys_WriteStderr("\n");
917 /* Back-up one character */
920 tok_backup(register struct tok_state
*tok
, register int c
)
923 if (--tok
->cur
< tok
->buf
)
924 Py_FatalError("tok_backup: begin of buffer");
931 /* Return the token corresponding to a single character */
934 PyToken_OneChar(int c
)
937 case '(': return LPAR
;
938 case ')': return RPAR
;
939 case '[': return LSQB
;
940 case ']': return RSQB
;
941 case ':': return COLON
;
942 case ',': return COMMA
;
943 case ';': return SEMI
;
944 case '+': return PLUS
;
945 case '-': return MINUS
;
946 case '*': return STAR
;
947 case '/': return SLASH
;
948 case '|': return VBAR
;
949 case '&': return AMPER
;
950 case '<': return LESS
;
951 case '>': return GREATER
;
952 case '=': return EQUAL
;
953 case '.': return DOT
;
954 case '%': return PERCENT
;
955 case '`': return BACKQUOTE
;
956 case '{': return LBRACE
;
957 case '}': return RBRACE
;
958 case '^': return CIRCUMFLEX
;
959 case '~': return TILDE
;
967 PyToken_TwoChars(int c1
, int c2
)
972 case '=': return EQEQUAL
;
977 case '=': return NOTEQUAL
;
982 case '>': return NOTEQUAL
;
983 case '=': return LESSEQUAL
;
984 case '<': return LEFTSHIFT
;
989 case '=': return GREATEREQUAL
;
990 case '>': return RIGHTSHIFT
;
995 case '=': return PLUSEQUAL
;
1000 case '=': return MINEQUAL
;
1005 case '*': return DOUBLESTAR
;
1006 case '=': return STAREQUAL
;
1011 case '/': return DOUBLESLASH
;
1012 case '=': return SLASHEQUAL
;
1017 case '=': return VBAREQUAL
;
1022 case '=': return PERCENTEQUAL
;
1027 case '=': return AMPEREQUAL
;
1032 case '=': return CIRCUMFLEXEQUAL
;
1040 PyToken_ThreeChars(int c1
, int c2
, int c3
)
1048 return LEFTSHIFTEQUAL
;
1058 return RIGHTSHIFTEQUAL
;
1068 return DOUBLESTAREQUAL
;
1078 return DOUBLESLASHEQUAL
;
1088 indenterror(struct tok_state
*tok
)
1090 if (tok
->alterror
) {
1091 tok
->done
= E_TABSPACE
;
1092 tok
->cur
= tok
->inp
;
1095 if (tok
->altwarning
) {
1096 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
1097 "in indentation\n", tok
->filename
);
1098 tok
->altwarning
= 0;
1104 /* Get next token, after space stripping etc. */
1107 tok_get(register struct tok_state
*tok
, char **p_start
, char **p_end
)
1112 *p_start
= *p_end
= NULL
;
1117 /* Get indentation level */
1119 register int col
= 0;
1120 register int altcol
= 0;
1126 else if (c
== '\t') {
1127 col
= (col
/tok
->tabsize
+ 1) * tok
->tabsize
;
1128 altcol
= (altcol
/tok
->alttabsize
+ 1)
1131 else if (c
== '\014') /* Control-L (formfeed) */
1132 col
= altcol
= 0; /* For Emacs users */
1137 if (c
== '#' || c
== '\n') {
1138 /* Lines with only whitespace and/or comments
1139 shouldn't affect the indentation and are
1140 not passed to the parser as NEWLINE tokens,
1141 except *totally* empty lines in interactive
1142 mode, which signal the end of a command group. */
1143 if (col
== 0 && c
== '\n' && tok
->prompt
!= NULL
)
1144 blankline
= 0; /* Let it through */
1146 blankline
= 1; /* Ignore completely */
1147 /* We can't jump back right here since we still
1148 may need to skip to the end of a comment */
1150 if (!blankline
&& tok
->level
== 0) {
1151 if (col
== tok
->indstack
[tok
->indent
]) {
1153 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1154 if (indenterror(tok
))
1158 else if (col
> tok
->indstack
[tok
->indent
]) {
1159 /* Indent -- always one */
1160 if (tok
->indent
+1 >= MAXINDENT
) {
1161 tok
->done
= E_TOODEEP
;
1162 tok
->cur
= tok
->inp
;
1165 if (altcol
<= tok
->altindstack
[tok
->indent
]) {
1166 if (indenterror(tok
))
1170 tok
->indstack
[++tok
->indent
] = col
;
1171 tok
->altindstack
[tok
->indent
] = altcol
;
1173 else /* col < tok->indstack[tok->indent] */ {
1174 /* Dedent -- any number, must be consistent */
1175 while (tok
->indent
> 0 &&
1176 col
< tok
->indstack
[tok
->indent
]) {
1180 if (col
!= tok
->indstack
[tok
->indent
]) {
1181 tok
->done
= E_DEDENT
;
1182 tok
->cur
= tok
->inp
;
1185 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1186 if (indenterror(tok
))
1193 tok
->start
= tok
->cur
;
1195 /* Return pending indents/dedents */
1196 if (tok
->pendin
!= 0) {
1197 if (tok
->pendin
< 0) {
1212 } while (c
== ' ' || c
== '\t' || c
== '\014');
1214 /* Set start of current token */
1215 tok
->start
= tok
->cur
- 1;
1217 /* Skip comment, while looking for tab-setting magic */
1219 static char *tabforms
[] = {
1220 "tab-width:", /* Emacs */
1221 ":tabstop=", /* vim, full form */
1222 ":ts=", /* vim, abbreviated form */
1223 "set tabsize=", /* will vi never die? */
1224 /* more templates can be added here to support other editors */
1230 *tp
++ = c
= tok_nextc(tok
);
1231 } while (c
!= EOF
&& c
!= '\n' &&
1232 tp
- cbuf
+ 1 < sizeof(cbuf
));
1235 cp
< tabforms
+ sizeof(tabforms
)/sizeof(tabforms
[0]);
1237 if ((tp
= strstr(cbuf
, *cp
))) {
1238 int newsize
= atoi(tp
+ strlen(*cp
));
1240 if (newsize
>= 1 && newsize
<= 40) {
1241 tok
->tabsize
= newsize
;
1244 "Tab size set to %d\n",
1249 while (c
!= EOF
&& c
!= '\n')
1253 /* Check for EOF and errors now */
1255 return tok
->done
== E_EOF
? ENDMARKER
: ERRORTOKEN
;
1258 /* Identifier (most frequent token!) */
1259 if (isalpha(c
) || c
== '_') {
1260 /* Process r"", u"" and ur"" */
1265 if (c
== '"' || c
== '\'')
1271 if (c
== 'r' || c
== 'R')
1273 if (c
== '"' || c
== '\'')
1277 while (isalnum(c
) || c
== '_') {
1281 *p_start
= tok
->start
;
1289 if (blankline
|| tok
->level
> 0)
1291 *p_start
= tok
->start
;
1292 *p_end
= tok
->cur
- 1; /* Leave '\n' out of the string */
1297 /* Period or number starting with period? */
1305 *p_start
= tok
->start
;
1314 /* Hex or octal -- maybe. */
1318 #ifndef WITHOUT_COMPLEX
1319 if (c
== 'j' || c
== 'J')
1322 if (c
== 'x' || c
== 'X') {
1326 } while (isxdigit(c
));
1329 int found_decimal
= 0;
1330 /* Octal; c is first char of it */
1331 /* There's no 'isoctdigit' macro, sigh */
1332 while ('0' <= c
&& c
< '8') {
1339 } while (isdigit(c
));
1343 else if (c
== 'e' || c
== 'E')
1345 #ifndef WITHOUT_COMPLEX
1346 else if (c
== 'j' || c
== 'J')
1349 else if (found_decimal
) {
1350 tok
->done
= E_TOKEN
;
1355 if (c
== 'l' || c
== 'L')
1362 } while (isdigit(c
));
1363 if (c
== 'l' || c
== 'L')
1366 /* Accept floating point numbers. */
1372 } while (isdigit(c
));
1374 if (c
== 'e' || c
== 'E') {
1378 if (c
== '+' || c
== '-')
1381 tok
->done
= E_TOKEN
;
1387 } while (isdigit(c
));
1389 #ifndef WITHOUT_COMPLEX
1390 if (c
== 'j' || c
== 'J')
1391 /* Imaginary part */
1398 *p_start
= tok
->start
;
1405 if (c
== '\'' || c
== '"') {
1406 Py_ssize_t quote2
= tok
->cur
- tok
->start
+ 1;
1419 tok
->cont_line
= 1; /* multiline string. */
1421 else if (c
== EOF
) {
1426 tok
->cur
= tok
->inp
;
1429 else if (c
== quote
) {
1431 if (tok
->cur
- tok
->start
== quote2
) {
1440 if (!triple
|| tripcount
== 3)
1443 else if (c
== '\\') {
1448 tok
->cur
= tok
->inp
;
1455 *p_start
= tok
->start
;
1460 /* Line continuation */
1464 tok
->done
= E_LINECONT
;
1465 tok
->cur
= tok
->inp
;
1469 goto again
; /* Read next line */
1472 /* Check for two-character token */
1474 int c2
= tok_nextc(tok
);
1475 int token
= PyToken_TwoChars(c
, c2
);
1477 int c3
= tok_nextc(tok
);
1478 int token3
= PyToken_ThreeChars(c
, c2
, c3
);
1482 tok_backup(tok
, c3
);
1484 *p_start
= tok
->start
;
1488 tok_backup(tok
, c2
);
1491 /* Keep track of parentheses nesting level */
1505 /* Punctuation character */
1506 *p_start
= tok
->start
;
1508 return PyToken_OneChar(c
);
1512 PyTokenizer_Get(struct tok_state
*tok
, char **p_start
, char **p_end
)
1514 int result
= tok_get(tok
, p_start
, p_end
);
1515 if (tok
->decoding_erred
) {
1516 result
= ERRORTOKEN
;
1517 tok
->done
= E_DECODE
;
1525 tok_dump(int type
, char *start
, char *end
)
1527 printf("%s", _PyParser_TokenNames
[type
]);
1528 if (type
== NAME
|| type
== NUMBER
|| type
== STRING
|| type
== OP
)
1529 printf("(%.*s)", (int)(end
- start
), start
);