2 /* Tokenizer implementation */
5 #include "pgenheaders.h"
10 #include "tokenizer.h"
14 #include "unicodeobject.h"
15 #include "stringobject.h"
16 #include "fileobject.h"
22 extern char *PyOS_Readline(FILE *, FILE *, char *);
23 /* Return malloc'ed string including trailing \n;
24 empty malloc'ed string for EOF;
25 NULL if interrupted */
27 /* Don't ever change this -- it would break the portability of Python code */
30 /* Convert a possibly signed character to a nonnegative int */
31 /* XXX This assumes characters are 8 bits wide */
32 #ifdef __CHAR_UNSIGNED__
33 #define Py_CHARMASK(c) (c)
35 #define Py_CHARMASK(c) ((c) & 0xff)
39 static struct tok_state
*tok_new(void);
40 static int tok_nextc(struct tok_state
*tok
);
41 static void tok_backup(struct tok_state
*tok
, int c
);
45 char *_PyParser_TokenNames
[] = {
97 /* This table must match the #defines in token.h! */
104 /* Create and initialize a new tok_state structure */
106 static struct tok_state
*
109 struct tok_state
*tok
= (struct tok_state
*)PyMem_MALLOC(
110 sizeof(struct tok_state
));
113 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= tok
->start
= NULL
;
116 tok
->tabsize
= TABSIZE
;
118 tok
->indstack
[0] = 0;
121 tok
->prompt
= tok
->nextprompt
= NULL
;
124 tok
->filename
= NULL
;
128 tok
->altindstack
[0] = 0;
129 tok
->decoding_state
= 0;
130 tok
->decoding_erred
= 0;
131 tok
->read_coding_spec
= 0;
132 tok
->encoding
= NULL
;
135 tok
->decoding_readline
= NULL
;
136 tok
->decoding_buffer
= NULL
;
144 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
146 return fgets(s
, size
, tok
->fp
);
150 decoding_feof(struct tok_state
*tok
)
152 return feof(tok
->fp
);
156 decode_str(const char *str
, struct tok_state
*tok
)
164 error_ret(struct tok_state
*tok
) /* XXX */
166 tok
->decoding_erred
= 1;
167 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
) /* see PyTokenizer_Free */
168 PyMem_FREE(tok
->buf
);
170 return NULL
; /* as if it were EOF */
174 new_string(const char *s
, Py_ssize_t len
)
176 char* result
= (char *)PyMem_MALLOC(len
+ 1);
177 if (result
!= NULL
) {
178 memcpy(result
, s
, len
);
185 get_normal_name(char *s
) /* for utf-8 and latin-1 */
189 for (i
= 0; i
< 12; i
++) {
191 if (c
== '\0') break;
192 else if (c
== '_') buf
[i
] = '-';
193 else buf
[i
] = tolower(c
);
196 if (strcmp(buf
, "utf-8") == 0 ||
197 strncmp(buf
, "utf-8-", 6) == 0) return "utf-8";
198 else if (strcmp(buf
, "latin-1") == 0 ||
199 strcmp(buf
, "iso-8859-1") == 0 ||
200 strcmp(buf
, "iso-latin-1") == 0 ||
201 strncmp(buf
, "latin-1-", 8) == 0 ||
202 strncmp(buf
, "iso-8859-1-", 11) == 0 ||
203 strncmp(buf
, "iso-latin-1-", 12) == 0) return "iso-8859-1";
207 /* Return the coding spec in S, or NULL if none is found. */
210 get_coding_spec(const char *s
, Py_ssize_t size
)
213 /* Coding spec must be in a comment, and that comment must be
214 * the only statement on the source code line. */
215 for (i
= 0; i
< size
- 6; i
++) {
218 if (s
[i
] != ' ' && s
[i
] != '\t' && s
[i
] != '\014')
221 for (; i
< size
- 6; i
++) { /* XXX inefficient search */
222 const char* t
= s
+ i
;
223 if (strncmp(t
, "coding", 6) == 0) {
224 const char* begin
= NULL
;
226 if (t
[0] != ':' && t
[0] != '=')
230 } while (t
[0] == '\x20' || t
[0] == '\t');
233 while (isalnum(Py_CHARMASK(t
[0])) ||
234 t
[0] == '-' || t
[0] == '_' || t
[0] == '.')
238 char* r
= new_string(begin
, t
- begin
);
239 char* q
= get_normal_name(r
);
242 r
= new_string(q
, strlen(q
));
251 /* Check whether the line contains a coding spec. If it does,
252 invoke the set_readline function for the new encoding.
253 This function receives the tok_state and the new encoding.
254 Return 1 on success, 0 on failure. */
257 check_coding_spec(const char* line
, Py_ssize_t size
, struct tok_state
*tok
,
258 int set_readline(struct tok_state
*, const char *))
264 /* It's a continuation line, so it can't be a coding spec. */
266 cs
= get_coding_spec(line
, size
);
268 tok
->read_coding_spec
= 1;
269 if (tok
->encoding
== NULL
) {
270 assert(tok
->decoding_state
== 1); /* raw */
271 if (strcmp(cs
, "utf-8") == 0 ||
272 strcmp(cs
, "iso-8859-1") == 0) {
275 #ifdef Py_USING_UNICODE
276 r
= set_readline(tok
, cs
);
279 tok
->decoding_state
= -1;
284 /* Without Unicode support, we cannot
285 process the coding spec. Since there
286 won't be any Unicode literals, that
291 } else { /* then, compare cs with BOM */
292 r
= (strcmp(tok
->encoding
, cs
) == 0);
300 PyErr_Format(PyExc_SyntaxError
, "encoding problem: %s", cs
);
305 /* See whether the file starts with a BOM. If it does,
306 invoke the set_readline function with the new encoding.
307 Return 1 on success, 0 on failure. */
310 check_bom(int get_char(struct tok_state
*),
311 void unget_char(int, struct tok_state
*),
312 int set_readline(struct tok_state
*, const char *),
313 struct tok_state
*tok
)
315 int ch
= get_char(tok
);
316 tok
->decoding_state
= 1;
319 } else if (ch
== 0xEF) {
320 ch
= get_char(tok
); if (ch
!= 0xBB) goto NON_BOM
;
321 ch
= get_char(tok
); if (ch
!= 0xBF) goto NON_BOM
;
323 /* Disable support for UTF-16 BOMs until a decision
324 is made whether this needs to be supported. */
325 } else if (ch
== 0xFE) {
326 ch
= get_char(tok
); if (ch
!= 0xFF) goto NON_BOM
;
327 if (!set_readline(tok
, "utf-16-be")) return 0;
328 tok
->decoding_state
= -1;
329 } else if (ch
== 0xFF) {
330 ch
= get_char(tok
); if (ch
!= 0xFE) goto NON_BOM
;
331 if (!set_readline(tok
, "utf-16-le")) return 0;
332 tok
->decoding_state
= -1;
338 if (tok
->encoding
!= NULL
)
339 PyMem_FREE(tok
->encoding
);
340 tok
->encoding
= new_string("utf-8", 5); /* resulting is in utf-8 */
343 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
344 unget_char(0xFF, tok
); /* XXX this will cause a syntax error */
348 /* Read a line of text from TOK into S, using the stream in TOK.
349 Return NULL on failure, else S.
351 On entry, tok->decoding_buffer will be one of:
352 1) NULL: need to call tok->decoding_readline to get a new line
353 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
354 stored the result in tok->decoding_buffer
355 3) PyStringObject *: previous call to fp_readl did not have enough room
356 (in the s buffer) to copy entire contents of the line read
357 by tok->decoding_readline. tok->decoding_buffer has the overflow.
358 In this case, fp_readl is called in a loop (with an expanded buffer)
359 until the buffer ends with a '\n' (or until the end of the file is
360 reached): see tok_nextc and its calls to decoding_fgets.
364 fp_readl(char *s
, int size
, struct tok_state
*tok
)
366 #ifndef Py_USING_UNICODE
367 /* In a non-Unicode built, this should never be called. */
368 Py_FatalError("fp_readl should not be called in this build.");
369 return NULL
; /* Keep compiler happy (not reachable) */
371 PyObject
* utf8
= NULL
;
372 PyObject
* buf
= tok
->decoding_buffer
;
376 /* Ask for one less byte so we can terminate it */
381 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
383 return error_ret(tok
);
385 tok
->decoding_buffer
= NULL
;
386 if (PyString_CheckExact(buf
))
390 utf8
= PyUnicode_AsUTF8String(buf
);
393 return error_ret(tok
);
395 str
= PyString_AsString(utf8
);
396 utf8len
= PyString_GET_SIZE(utf8
);
397 if (utf8len
> size
) {
398 tok
->decoding_buffer
= PyString_FromStringAndSize(str
+size
, utf8len
-size
);
399 if (tok
->decoding_buffer
== NULL
) {
401 return error_ret(tok
);
405 memcpy(s
, str
, utf8len
);
408 if (utf8len
== 0) return NULL
; /* EOF */
413 /* Set the readline function for TOK to a StreamReader's
414 readline function. The StreamReader is named ENC.
416 This function is called from check_bom and check_coding_spec.
418 ENC is usually identical to the future value of tok->encoding,
419 except for the (currently unsupported) case of UTF-16.
421 Return 1 on success, 0 on failure. */
424 fp_setreadl(struct tok_state
*tok
, const char* enc
)
426 PyObject
*reader
, *stream
, *readline
;
428 /* XXX: constify filename argument. */
429 stream
= PyFile_FromFile(tok
->fp
, (char*)tok
->filename
, "rb", NULL
);
433 reader
= PyCodec_StreamReader(enc
, stream
, NULL
);
438 readline
= PyObject_GetAttrString(reader
, "readline");
440 if (readline
== NULL
)
443 tok
->decoding_readline
= readline
;
447 /* Fetch the next byte from TOK. */
449 static int fp_getc(struct tok_state
*tok
) {
450 return getc(tok
->fp
);
453 /* Unfetch the last byte back into TOK. */
455 static void fp_ungetc(int c
, struct tok_state
*tok
) {
459 /* Read a line of input from TOK. Determine encoding
463 decoding_fgets(char *s
, int size
, struct tok_state
*tok
)
468 if (tok
->decoding_state
< 0) {
469 /* We already have a codec associated with
471 line
= fp_readl(s
, size
, tok
);
473 } else if (tok
->decoding_state
> 0) {
474 /* We want a 'raw' read. */
475 line
= Py_UniversalNewlineFgets(s
, size
,
479 /* We have not yet determined the encoding.
480 If an encoding is found, use the file-pointer
481 reader functions from now on. */
482 if (!check_bom(fp_getc
, fp_ungetc
, fp_setreadl
, tok
))
483 return error_ret(tok
);
484 assert(tok
->decoding_state
!= 0);
487 if (line
!= NULL
&& tok
->lineno
< 2 && !tok
->read_coding_spec
) {
488 if (!check_coding_spec(line
, strlen(line
), tok
, fp_setreadl
)) {
489 return error_ret(tok
);
493 /* The default encoding is ASCII, so make sure we don't have any
494 non-ASCII bytes in it. */
495 if (line
&& !tok
->encoding
) {
497 for (c
= (unsigned char *)line
; *c
; c
++)
505 /* Need to add 1 to the line number, since this line
506 has not been counted, yet. */
508 "Non-ASCII character '\\x%.2x' "
509 "in file %.200s on line %i, "
510 "but no encoding declared; "
511 "see http://www.python.org/peps/pep-0263.html for details",
512 badchar
, tok
->filename
, tok
->lineno
+ 1);
513 PyErr_SetString(PyExc_SyntaxError
, buf
);
514 return error_ret(tok
);
521 decoding_feof(struct tok_state
*tok
)
523 if (tok
->decoding_state
>= 0) {
524 return feof(tok
->fp
);
526 PyObject
* buf
= tok
->decoding_buffer
;
528 buf
= PyObject_CallObject(tok
->decoding_readline
, NULL
);
533 tok
->decoding_buffer
= buf
;
536 return PyObject_Length(buf
) == 0;
540 /* Fetch a byte from TOK, using the string buffer. */
543 buf_getc(struct tok_state
*tok
) {
544 return Py_CHARMASK(*tok
->str
++);
547 /* Unfetch a byte from TOK, using the string buffer. */
550 buf_ungetc(int c
, struct tok_state
*tok
) {
552 assert(Py_CHARMASK(*tok
->str
) == c
); /* tok->cur may point to read-only segment */
555 /* Set the readline function for TOK to ENC. For the string-based
556 tokenizer, this means to just record the encoding. */
559 buf_setreadl(struct tok_state
*tok
, const char* enc
) {
564 /* Return a UTF-8 encoding Python string object from the
565 C byte string STR, which is encoded with ENC. */
567 #ifdef Py_USING_UNICODE
569 translate_into_utf8(const char* str
, const char* enc
) {
571 PyObject
* buf
= PyUnicode_Decode(str
, strlen(str
), enc
, NULL
);
574 utf8
= PyUnicode_AsUTF8String(buf
);
580 /* Decode a byte string STR for use as the buffer of TOK.
581 Look for encoding declarations inside STR, and record them
585 decode_str(const char *str
, struct tok_state
*tok
)
587 PyObject
* utf8
= NULL
;
592 if (!check_bom(buf_getc
, buf_ungetc
, buf_setreadl
, tok
))
593 return error_ret(tok
);
594 str
= tok
->str
; /* string after BOM if any */
596 #ifdef Py_USING_UNICODE
597 if (tok
->enc
!= NULL
) {
598 utf8
= translate_into_utf8(str
, tok
->enc
);
600 return error_ret(tok
);
601 str
= PyString_AsString(utf8
);
604 for (s
= str
;; s
++) {
605 if (*s
== '\0') break;
606 else if (*s
== '\n') {
608 if (lineno
== 2) break;
612 if (!check_coding_spec(str
, s
- str
, tok
, buf_setreadl
))
613 return error_ret(tok
);
614 #ifdef Py_USING_UNICODE
615 if (tok
->enc
!= NULL
) {
616 assert(utf8
== NULL
);
617 utf8
= translate_into_utf8(str
, tok
->enc
);
619 PyErr_Format(PyExc_SyntaxError
,
620 "unknown encoding: %s", tok
->enc
);
621 return error_ret(tok
);
623 str
= PyString_AsString(utf8
);
626 assert(tok
->decoding_buffer
== NULL
);
627 tok
->decoding_buffer
= utf8
; /* CAUTION */
633 /* Set up tokenizer for string */
636 PyTokenizer_FromString(const char *str
)
638 struct tok_state
*tok
= tok_new();
641 str
= (char *)decode_str(str
, tok
);
643 PyTokenizer_Free(tok
);
647 /* XXX: constify members. */
648 tok
->buf
= tok
->cur
= tok
->end
= tok
->inp
= (char*)str
;
653 /* Set up tokenizer for file */
656 PyTokenizer_FromFile(FILE *fp
, char *ps1
, char *ps2
)
658 struct tok_state
*tok
= tok_new();
661 if ((tok
->buf
= (char *)PyMem_MALLOC(BUFSIZ
)) == NULL
) {
662 PyTokenizer_Free(tok
);
665 tok
->cur
= tok
->inp
= tok
->buf
;
666 tok
->end
= tok
->buf
+ BUFSIZ
;
669 tok
->nextprompt
= ps2
;
674 /* Free a tok_state structure */
677 PyTokenizer_Free(struct tok_state
*tok
)
679 if (tok
->encoding
!= NULL
)
680 PyMem_FREE(tok
->encoding
);
682 Py_XDECREF(tok
->decoding_readline
);
683 Py_XDECREF(tok
->decoding_buffer
);
685 if (tok
->fp
!= NULL
&& tok
->buf
!= NULL
)
686 PyMem_FREE(tok
->buf
);
690 #if !defined(PGEN) && defined(Py_USING_UNICODE)
692 tok_stdin_decode(struct tok_state
*tok
, char **inp
)
694 PyObject
*enc
, *sysstdin
, *decoded
, *utf8
;
695 const char *encoding
;
698 if (PySys_GetFile((char *)"stdin", NULL
) != stdin
)
700 sysstdin
= PySys_GetObject("stdin");
701 if (sysstdin
== NULL
|| !PyFile_Check(sysstdin
))
704 enc
= ((PyFileObject
*)sysstdin
)->f_encoding
;
705 if (enc
== NULL
|| !PyString_Check(enc
))
709 encoding
= PyString_AsString(enc
);
710 decoded
= PyUnicode_Decode(*inp
, strlen(*inp
), encoding
, NULL
);
714 utf8
= PyUnicode_AsEncodedString(decoded
, "utf-8", NULL
);
719 assert(PyString_Check(utf8
));
720 converted
= new_string(PyString_AS_STRING(utf8
),
721 PyString_GET_SIZE(utf8
));
723 if (converted
== NULL
)
728 if (tok
->encoding
!= NULL
)
729 PyMem_FREE(tok
->encoding
);
730 tok
->encoding
= new_string(encoding
, strlen(encoding
));
731 if (tok
->encoding
== NULL
)
743 /* Fallback to iso-8859-1: for backward compatibility */
750 /* Get next char, updating state; error code goes into tok->done */
753 tok_nextc(register struct tok_state
*tok
)
756 if (tok
->cur
!= tok
->inp
) {
757 return Py_CHARMASK(*tok
->cur
++); /* Fast path */
759 if (tok
->done
!= E_OK
)
761 if (tok
->fp
== NULL
) {
762 char *end
= strchr(tok
->inp
, '\n');
766 end
= strchr(tok
->inp
, '\0');
767 if (end
== tok
->inp
) {
772 if (tok
->start
== NULL
)
774 tok
->line_start
= tok
->cur
;
777 return Py_CHARMASK(*tok
->cur
++);
779 if (tok
->prompt
!= NULL
) {
780 char *newtok
= PyOS_Readline(stdin
, stdout
, tok
->prompt
);
781 if (tok
->nextprompt
!= NULL
)
782 tok
->prompt
= tok
->nextprompt
;
785 else if (*newtok
== '\0') {
789 #if !defined(PGEN) && defined(Py_USING_UNICODE)
790 else if (tok_stdin_decode(tok
, &newtok
) != 0)
793 else if (tok
->start
!= NULL
) {
794 size_t start
= tok
->start
- tok
->buf
;
795 size_t oldlen
= tok
->cur
- tok
->buf
;
796 size_t newlen
= oldlen
+ strlen(newtok
);
797 char *buf
= tok
->buf
;
798 buf
= (char *)PyMem_REALLOC(buf
, newlen
+1);
801 PyMem_FREE(tok
->buf
);
808 tok
->cur
= tok
->buf
+ oldlen
;
809 tok
->line_start
= tok
->cur
;
810 strcpy(tok
->buf
+ oldlen
, newtok
);
812 tok
->inp
= tok
->buf
+ newlen
;
813 tok
->end
= tok
->inp
+ 1;
814 tok
->start
= tok
->buf
+ start
;
818 if (tok
->buf
!= NULL
)
819 PyMem_FREE(tok
->buf
);
821 tok
->line_start
= tok
->buf
;
823 tok
->line_start
= tok
->buf
;
824 tok
->inp
= strchr(tok
->buf
, '\0');
825 tok
->end
= tok
->inp
+ 1;
832 if (tok
->start
== NULL
) {
833 if (tok
->buf
== NULL
) {
835 PyMem_MALLOC(BUFSIZ
);
836 if (tok
->buf
== NULL
) {
840 tok
->end
= tok
->buf
+ BUFSIZ
;
842 if (decoding_fgets(tok
->buf
, (int)(tok
->end
- tok
->buf
),
849 tok
->inp
= strchr(tok
->buf
, '\0');
850 done
= tok
->inp
[-1] == '\n';
854 cur
= tok
->cur
- tok
->buf
;
855 if (decoding_feof(tok
)) {
863 /* Read until '\n' or EOF */
865 Py_ssize_t curstart
= tok
->start
== NULL
? -1 :
866 tok
->start
- tok
->buf
;
867 Py_ssize_t curvalid
= tok
->inp
- tok
->buf
;
868 Py_ssize_t newsize
= curvalid
+ BUFSIZ
;
869 char *newbuf
= tok
->buf
;
870 newbuf
= (char *)PyMem_REALLOC(newbuf
,
872 if (newbuf
== NULL
) {
878 tok
->inp
= tok
->buf
+ curvalid
;
879 tok
->end
= tok
->buf
+ newsize
;
880 tok
->start
= curstart
< 0 ? NULL
:
882 if (decoding_fgets(tok
->inp
,
883 (int)(tok
->end
- tok
->inp
),
885 /* Break out early on decoding
886 errors, as tok->buf will be NULL
888 if (tok
->decoding_erred
)
890 /* Last line does not end in \n,
892 strcpy(tok
->inp
, "\n");
894 tok
->inp
= strchr(tok
->inp
, '\0');
895 done
= tok
->inp
[-1] == '\n';
897 if (tok
->buf
!= NULL
) {
898 tok
->cur
= tok
->buf
+ cur
;
899 tok
->line_start
= tok
->cur
;
900 /* replace "\r\n" with "\n" */
901 /* For Mac leave the \r, giving a syntax error */
903 if (pt
>= tok
->buf
&& *pt
== '\r') {
910 if (tok
->done
!= E_OK
) {
911 if (tok
->prompt
!= NULL
)
912 PySys_WriteStderr("\n");
921 /* Back-up one character */
924 tok_backup(register struct tok_state
*tok
, register int c
)
927 if (--tok
->cur
< tok
->buf
)
928 Py_FatalError("tok_backup: begin of buffer");
935 /* Return the token corresponding to a single character */
938 PyToken_OneChar(int c
)
941 case '(': return LPAR
;
942 case ')': return RPAR
;
943 case '[': return LSQB
;
944 case ']': return RSQB
;
945 case ':': return COLON
;
946 case ',': return COMMA
;
947 case ';': return SEMI
;
948 case '+': return PLUS
;
949 case '-': return MINUS
;
950 case '*': return STAR
;
951 case '/': return SLASH
;
952 case '|': return VBAR
;
953 case '&': return AMPER
;
954 case '<': return LESS
;
955 case '>': return GREATER
;
956 case '=': return EQUAL
;
957 case '.': return DOT
;
958 case '%': return PERCENT
;
959 case '`': return BACKQUOTE
;
960 case '{': return LBRACE
;
961 case '}': return RBRACE
;
962 case '^': return CIRCUMFLEX
;
963 case '~': return TILDE
;
971 PyToken_TwoChars(int c1
, int c2
)
976 case '=': return EQEQUAL
;
981 case '=': return NOTEQUAL
;
986 case '>': return NOTEQUAL
;
987 case '=': return LESSEQUAL
;
988 case '<': return LEFTSHIFT
;
993 case '=': return GREATEREQUAL
;
994 case '>': return RIGHTSHIFT
;
999 case '=': return PLUSEQUAL
;
1004 case '=': return MINEQUAL
;
1009 case '*': return DOUBLESTAR
;
1010 case '=': return STAREQUAL
;
1015 case '/': return DOUBLESLASH
;
1016 case '=': return SLASHEQUAL
;
1021 case '=': return VBAREQUAL
;
1026 case '=': return PERCENTEQUAL
;
1031 case '=': return AMPEREQUAL
;
1036 case '=': return CIRCUMFLEXEQUAL
;
1044 PyToken_ThreeChars(int c1
, int c2
, int c3
)
1052 return LEFTSHIFTEQUAL
;
1062 return RIGHTSHIFTEQUAL
;
1072 return DOUBLESTAREQUAL
;
1082 return DOUBLESLASHEQUAL
;
1092 indenterror(struct tok_state
*tok
)
1094 if (tok
->alterror
) {
1095 tok
->done
= E_TABSPACE
;
1096 tok
->cur
= tok
->inp
;
1099 if (tok
->altwarning
) {
1100 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
1101 "in indentation\n", tok
->filename
);
1102 tok
->altwarning
= 0;
1108 /* Get next token, after space stripping etc. */
1111 tok_get(register struct tok_state
*tok
, char **p_start
, char **p_end
)
1116 *p_start
= *p_end
= NULL
;
1121 /* Get indentation level */
1123 register int col
= 0;
1124 register int altcol
= 0;
1130 else if (c
== '\t') {
1131 col
= (col
/tok
->tabsize
+ 1) * tok
->tabsize
;
1132 altcol
= (altcol
/tok
->alttabsize
+ 1)
1135 else if (c
== '\014') /* Control-L (formfeed) */
1136 col
= altcol
= 0; /* For Emacs users */
1141 if (c
== '#' || c
== '\n') {
1142 /* Lines with only whitespace and/or comments
1143 shouldn't affect the indentation and are
1144 not passed to the parser as NEWLINE tokens,
1145 except *totally* empty lines in interactive
1146 mode, which signal the end of a command group. */
1147 if (col
== 0 && c
== '\n' && tok
->prompt
!= NULL
)
1148 blankline
= 0; /* Let it through */
1150 blankline
= 1; /* Ignore completely */
1151 /* We can't jump back right here since we still
1152 may need to skip to the end of a comment */
1154 if (!blankline
&& tok
->level
== 0) {
1155 if (col
== tok
->indstack
[tok
->indent
]) {
1157 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1158 if (indenterror(tok
))
1162 else if (col
> tok
->indstack
[tok
->indent
]) {
1163 /* Indent -- always one */
1164 if (tok
->indent
+1 >= MAXINDENT
) {
1165 tok
->done
= E_TOODEEP
;
1166 tok
->cur
= tok
->inp
;
1169 if (altcol
<= tok
->altindstack
[tok
->indent
]) {
1170 if (indenterror(tok
))
1174 tok
->indstack
[++tok
->indent
] = col
;
1175 tok
->altindstack
[tok
->indent
] = altcol
;
1177 else /* col < tok->indstack[tok->indent] */ {
1178 /* Dedent -- any number, must be consistent */
1179 while (tok
->indent
> 0 &&
1180 col
< tok
->indstack
[tok
->indent
]) {
1184 if (col
!= tok
->indstack
[tok
->indent
]) {
1185 tok
->done
= E_DEDENT
;
1186 tok
->cur
= tok
->inp
;
1189 if (altcol
!= tok
->altindstack
[tok
->indent
]) {
1190 if (indenterror(tok
))
1197 tok
->start
= tok
->cur
;
1199 /* Return pending indents/dedents */
1200 if (tok
->pendin
!= 0) {
1201 if (tok
->pendin
< 0) {
1216 } while (c
== ' ' || c
== '\t' || c
== '\014');
1218 /* Set start of current token */
1219 tok
->start
= tok
->cur
- 1;
1221 /* Skip comment, while looking for tab-setting magic */
1223 static char *tabforms
[] = {
1224 "tab-width:", /* Emacs */
1225 ":tabstop=", /* vim, full form */
1226 ":ts=", /* vim, abbreviated form */
1227 "set tabsize=", /* will vi never die? */
1228 /* more templates can be added here to support other editors */
1234 *tp
++ = c
= tok_nextc(tok
);
1235 } while (c
!= EOF
&& c
!= '\n' &&
1236 (size_t)(tp
- cbuf
+ 1) < sizeof(cbuf
));
1239 cp
< tabforms
+ sizeof(tabforms
)/sizeof(tabforms
[0]);
1241 if ((tp
= strstr(cbuf
, *cp
))) {
1242 int newsize
= atoi(tp
+ strlen(*cp
));
1244 if (newsize
>= 1 && newsize
<= 40) {
1245 tok
->tabsize
= newsize
;
1248 "Tab size set to %d\n",
1253 while (c
!= EOF
&& c
!= '\n')
1257 /* Check for EOF and errors now */
1259 return tok
->done
== E_EOF
? ENDMARKER
: ERRORTOKEN
;
1262 /* Identifier (most frequent token!) */
1263 if (isalpha(c
) || c
== '_') {
1264 /* Process r"", u"" and ur"" */
1269 if (c
== '"' || c
== '\'')
1275 if (c
== 'r' || c
== 'R')
1277 if (c
== '"' || c
== '\'')
1281 while (isalnum(c
) || c
== '_') {
1285 *p_start
= tok
->start
;
1293 if (blankline
|| tok
->level
> 0)
1295 *p_start
= tok
->start
;
1296 *p_end
= tok
->cur
- 1; /* Leave '\n' out of the string */
1301 /* Period or number starting with period? */
1309 *p_start
= tok
->start
;
1318 /* Hex or octal -- maybe. */
1322 #ifndef WITHOUT_COMPLEX
1323 if (c
== 'j' || c
== 'J')
1326 if (c
== 'x' || c
== 'X') {
1330 } while (isxdigit(c
));
1333 int found_decimal
= 0;
1334 /* Octal; c is first char of it */
1335 /* There's no 'isoctdigit' macro, sigh */
1336 while ('0' <= c
&& c
< '8') {
1343 } while (isdigit(c
));
1347 else if (c
== 'e' || c
== 'E')
1349 #ifndef WITHOUT_COMPLEX
1350 else if (c
== 'j' || c
== 'J')
1353 else if (found_decimal
) {
1354 tok
->done
= E_TOKEN
;
1359 if (c
== 'l' || c
== 'L')
1366 } while (isdigit(c
));
1367 if (c
== 'l' || c
== 'L')
1370 /* Accept floating point numbers. */
1376 } while (isdigit(c
));
1378 if (c
== 'e' || c
== 'E') {
1382 if (c
== '+' || c
== '-')
1385 tok
->done
= E_TOKEN
;
1391 } while (isdigit(c
));
1393 #ifndef WITHOUT_COMPLEX
1394 if (c
== 'j' || c
== 'J')
1395 /* Imaginary part */
1402 *p_start
= tok
->start
;
1409 if (c
== '\'' || c
== '"') {
1410 Py_ssize_t quote2
= tok
->cur
- tok
->start
+ 1;
1423 tok
->cont_line
= 1; /* multiline string. */
1425 else if (c
== EOF
) {
1430 tok
->cur
= tok
->inp
;
1433 else if (c
== quote
) {
1435 if (tok
->cur
- tok
->start
== quote2
) {
1444 if (!triple
|| tripcount
== 3)
1447 else if (c
== '\\') {
1452 tok
->cur
= tok
->inp
;
1459 *p_start
= tok
->start
;
1464 /* Line continuation */
1468 tok
->done
= E_LINECONT
;
1469 tok
->cur
= tok
->inp
;
1473 goto again
; /* Read next line */
1476 /* Check for two-character token */
1478 int c2
= tok_nextc(tok
);
1479 int token
= PyToken_TwoChars(c
, c2
);
1481 if (Py_Py3kWarningFlag
&& token
== NOTEQUAL
&& c
== '<') {
1482 if (PyErr_WarnExplicit(PyExc_DeprecationWarning
,
1483 "<> not supported in 3.x",
1484 tok
->filename
, tok
->lineno
,
1491 int c3
= tok_nextc(tok
);
1492 int token3
= PyToken_ThreeChars(c
, c2
, c3
);
1496 tok_backup(tok
, c3
);
1498 *p_start
= tok
->start
;
1502 tok_backup(tok
, c2
);
1505 /* Keep track of parentheses nesting level */
1519 /* Punctuation character */
1520 *p_start
= tok
->start
;
1522 return PyToken_OneChar(c
);
1526 PyTokenizer_Get(struct tok_state
*tok
, char **p_start
, char **p_end
)
1528 int result
= tok_get(tok
, p_start
, p_end
);
1529 if (tok
->decoding_erred
) {
1530 result
= ERRORTOKEN
;
1531 tok
->done
= E_DECODE
;
1536 /* This function is only called from parsetok. However, it cannot live
1537 there, as it must be empty for PGEN, and we can check for PGEN only
1542 PyTokenizer_RestoreEncoding(struct tok_state
* tok
, int len
, int* offset
)
1547 #ifdef Py_USING_UNICODE
1549 dec_utf8(const char *enc
, const char *text
, size_t len
) {
1550 PyObject
*ret
= NULL
;
1551 PyObject
*unicode_text
= PyUnicode_DecodeUTF8(text
, len
, "replace");
1553 ret
= PyUnicode_AsEncodedString(unicode_text
, enc
, "replace");
1554 Py_DECREF(unicode_text
);
1563 PyTokenizer_RestoreEncoding(struct tok_state
* tok
, int len
, int *offset
)
1566 if (tok
->encoding
) {
1567 /* convert source to original encondig */
1568 PyObject
*lineobj
= dec_utf8(tok
->encoding
, tok
->buf
, len
);
1569 if (lineobj
!= NULL
) {
1570 int linelen
= PyString_Size(lineobj
);
1571 const char *line
= PyString_AsString(lineobj
);
1572 text
= PyObject_MALLOC(linelen
+ 1);
1573 if (text
!= NULL
&& line
!= NULL
) {
1575 strncpy(text
, line
, linelen
);
1576 text
[linelen
] = '\0';
1580 /* adjust error offset */
1582 PyObject
*offsetobj
= dec_utf8(tok
->encoding
,
1583 tok
->buf
, *offset
-1);
1585 *offset
= PyString_Size(offsetobj
) + 1;
1586 Py_DECREF(offsetobj
);
1595 #endif /* defined(Py_USING_UNICODE) */
1602 tok_dump(int type
, char *start
, char *end
)
1604 printf("%s", _PyParser_TokenNames
[type
]);
1605 if (type
== NAME
|| type
== NUMBER
|| type
== STRING
|| type
== OP
)
1606 printf("(%.*s)", (int)(end
- start
), start
);