2 /* Parser-tokenizer link implementation */
4 #include "pgenheaders.h"
15 static node
*parsetok(struct tok_state
*, grammar
*, int, perrdetail
*, int *);
16 static void initerr(perrdetail
*err_ret
, const char* filename
);
18 /* Parse input coming from a string. Return error code, print some errors. */
20 PyParser_ParseString(const char *s
, grammar
*g
, int start
, perrdetail
*err_ret
)
22 return PyParser_ParseStringFlagsFilename(s
, NULL
, g
, start
, err_ret
, 0);
26 PyParser_ParseStringFlags(const char *s
, grammar
*g
, int start
,
27 perrdetail
*err_ret
, int flags
)
29 return PyParser_ParseStringFlagsFilename(s
, NULL
,
30 g
, start
, err_ret
, flags
);
34 PyParser_ParseStringFlagsFilename(const char *s
, const char *filename
,
35 grammar
*g
, int start
,
36 perrdetail
*err_ret
, int flags
)
39 return PyParser_ParseStringFlagsFilenameEx(s
, filename
, g
, start
,
44 PyParser_ParseStringFlagsFilenameEx(const char *s
, const char *filename
,
45 grammar
*g
, int start
,
46 perrdetail
*err_ret
, int *flags
)
48 struct tok_state
*tok
;
50 initerr(err_ret
, filename
);
52 if (*flags
& PyPARSE_IGNORE_COOKIE
)
53 tok
= PyTokenizer_FromUTF8(s
);
55 tok
= PyTokenizer_FromString(s
);
57 err_ret
->error
= PyErr_Occurred() ? E_DECODE
: E_NOMEM
;
61 tok
->filename
= filename
? filename
: "<string>";
62 return parsetok(tok
, g
, start
, err_ret
, flags
);
65 /* Parse input coming from a file. Return error code, print some errors. */
68 PyParser_ParseFile(FILE *fp
, const char *filename
, grammar
*g
, int start
,
69 char *ps1
, char *ps2
, perrdetail
*err_ret
)
71 return PyParser_ParseFileFlags(fp
, filename
, NULL
,
72 g
, start
, ps1
, ps2
, err_ret
, 0);
76 PyParser_ParseFileFlags(FILE *fp
, const char *filename
, const char *enc
,
77 grammar
*g
, int start
,
78 char *ps1
, char *ps2
, perrdetail
*err_ret
, int flags
)
81 return PyParser_ParseFileFlagsEx(fp
, filename
, enc
, g
, start
, ps1
,
82 ps2
, err_ret
, &iflags
);
86 PyParser_ParseFileFlagsEx(FILE *fp
, const char *filename
,
87 const char *enc
, grammar
*g
, int start
,
88 char *ps1
, char *ps2
, perrdetail
*err_ret
, int *flags
)
90 struct tok_state
*tok
;
92 initerr(err_ret
, filename
);
94 if ((tok
= PyTokenizer_FromFile(fp
, (char *)enc
, ps1
, ps2
)) == NULL
) {
95 err_ret
->error
= E_NOMEM
;
98 tok
->filename
= filename
;
99 return parsetok(tok
, g
, start
, err_ret
, flags
);
102 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
104 static char with_msg
[] =
105 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
107 static char as_msg
[] =
108 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
111 warn(const char *msg
, const char *filename
, int lineno
)
113 if (filename
== NULL
)
114 filename
= "<string>";
115 PySys_WriteStderr(msg
, filename
, lineno
);
120 /* Parse input coming from the given tokenizer structure.
121 Return error code. */
124 parsetok(struct tok_state
*tok
, grammar
*g
, int start
, perrdetail
*err_ret
,
129 int started
= 0, handling_import
= 0, handling_with
= 0;
131 if ((ps
= PyParser_New(g
, start
)) == NULL
) {
132 fprintf(stderr
, "no mem for new parser\n");
133 err_ret
->error
= E_NOMEM
;
134 PyTokenizer_Free(tok
);
137 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
138 if (*flags
& PyPARSE_BARRY_AS_BDFL
)
139 ps
->p_flags
|= CO_FUTURE_BARRY_AS_BDFL
;
149 type
= PyTokenizer_Get(tok
, &a
, &b
);
150 if (type
== ERRORTOKEN
) {
151 err_ret
->error
= tok
->done
;
154 if (type
== ENDMARKER
&& started
) {
155 type
= NEWLINE
; /* Add an extra newline */
156 handling_with
= handling_import
= 0;
158 /* Add the right number of dedent tokens,
159 except if a certain flag is given --
160 codeop.py uses this. */
162 !(*flags
& PyPARSE_DONT_IMPLY_DEDENT
))
164 tok
->pendin
= -tok
->indent
;
170 len
= b
- a
; /* XXX this may compute NULL - NULL */
171 str
= (char *) PyObject_MALLOC(len
+ 1);
173 fprintf(stderr
, "no mem for next token\n");
174 err_ret
->error
= E_NOMEM
;
178 strncpy(str
, a
, len
);
181 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
182 if (type
== NOTEQUAL
) {
183 if (!(ps
->p_flags
& CO_FUTURE_BARRY_AS_BDFL
) &&
185 err_ret
->error
= E_SYNTAX
;
188 else if ((ps
->p_flags
& CO_FUTURE_BARRY_AS_BDFL
) &&
190 err_ret
->text
= "with Barry as BDFL, use '<>' "
192 err_ret
->error
= E_SYNTAX
;
197 if (a
>= tok
->line_start
)
198 col_offset
= a
- tok
->line_start
;
202 if ((err_ret
->error
=
203 PyParser_AddToken(ps
, (int)type
, str
,
204 tok
->lineno
, col_offset
,
205 &(err_ret
->expected
))) != E_OK
) {
206 if (err_ret
->error
!= E_DONE
) {
208 err_ret
->token
= type
;
214 if (err_ret
->error
== E_DONE
) {
221 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
222 *flags
= ps
->p_flags
;
227 if (tok
->lineno
<= 1 && tok
->done
== E_EOF
)
228 err_ret
->error
= E_EOF
;
229 err_ret
->lineno
= tok
->lineno
;
230 if (tok
->buf
!= NULL
) {
232 assert(tok
->cur
- tok
->buf
< INT_MAX
);
233 err_ret
->offset
= (int)(tok
->cur
- tok
->buf
);
234 len
= tok
->inp
- tok
->buf
;
235 err_ret
->text
= (char *) PyObject_MALLOC(len
+ 1);
236 if (err_ret
->text
!= NULL
) {
238 strncpy(err_ret
->text
, tok
->buf
, len
);
239 err_ret
->text
[len
] = '\0';
242 } else if (tok
->encoding
!= NULL
) {
243 node
* r
= PyNode_New(encoding_decl
);
245 err_ret
->error
= E_NOMEM
;
249 r
->n_str
= tok
->encoding
;
252 tok
->encoding
= NULL
;
257 PyTokenizer_Free(tok
);
263 initerr(perrdetail
*err_ret
, const char *filename
)
265 err_ret
->error
= E_OK
;
266 err_ret
->filename
= filename
;
269 err_ret
->text
= NULL
;
271 err_ret
->expected
= -1;