Parser/parsetok.c

   1
   2 /* Parser-tokenizer link implementation */
   3
   4 #include "pgenheaders.h"
   5 #include "tokenizer.h"
   6 #include "node.h"
   7 #include "grammar.h"
   8 #include "parser.h"
   9 #include "parsetok.h"
  10 #include "errcode.h"
  11 #include "graminit.h"
  12
  13 int Py_TabcheckFlag;
  14
  15
  16 /* Forward */
  17 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
  18 static void initerr(perrdetail *err_ret, const char* filename);
  19
  20 /* Parse input coming from a string.  Return error code, print some errors. */
  21 node *
  22 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
  23 {
  24         return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
  25 }
  26
  27 node *
  28 PyParser_ParseStringFlags(const char *s, grammar *g, int start,
  29                           perrdetail *err_ret, int flags)
  30 {
  31         return PyParser_ParseStringFlagsFilename(s, NULL,
  32                                                  g, start, err_ret, flags);
  33 }
  34
  35 node *
  36 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
  37                           grammar *g, int start,
  38                           perrdetail *err_ret, int flags)
  39 {
  40         struct tok_state *tok;
  41
  42         initerr(err_ret, filename);
  43
  44         if ((tok = PyTokenizer_FromString(s)) == NULL) {
  45                 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
  46                 return NULL;
  47         }
  48
  49         tok->filename = filename ? filename : "<string>";
  50         if (Py_TabcheckFlag || Py_VerboseFlag) {
  51                 tok->altwarning = (tok->filename != NULL);
  52                 if (Py_TabcheckFlag >= 2)
  53                         tok->alterror++;
  54         }
  55
  56         return parsetok(tok, g, start, err_ret, flags);
  57 }
  58
  59 /* Parse input coming from a file.  Return error code, print some errors. */
  60
  61 node *
  62 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
  63                    char *ps1, char *ps2, perrdetail *err_ret)
  64 {
  65         return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
  66                                        err_ret, 0);
  67 }
  68
  69 node *
  70 PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
  71                         char *ps1, char *ps2, perrdetail *err_ret, int flags)
  72 {
  73         struct tok_state *tok;
  74
  75         initerr(err_ret, filename);
  76
  77         if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
  78                 err_ret->error = E_NOMEM;
  79                 return NULL;
  80         }
  81         tok->filename = filename;
  82         if (Py_TabcheckFlag || Py_VerboseFlag) {
  83                 tok->altwarning = (filename != NULL);
  84                 if (Py_TabcheckFlag >= 2)
  85                         tok->alterror++;
  86         }
  87
  88
  89         return parsetok(tok, g, start, err_ret, flags);
  90 }
  91
  92 /* Parse input coming from the given tokenizer structure.
  93    Return error code. */
  94
  95 #if 0 /* future keyword */
  96 static char yield_msg[] =
  97 "%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
  98 #endif
  99
 100 static node *
 101 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 102          int flags)
 103 {
 104         parser_state *ps;
 105         node *n;
 106         int started = 0;
 107
 108         if ((ps = PyParser_New(g, start)) == NULL) {
 109                 fprintf(stderr, "no mem for new parser\n");
 110                 err_ret->error = E_NOMEM;
 111                 PyTokenizer_Free(tok);
 112                 return NULL;
 113         }
 114 #if 0 /* future keyword */
 115         if (flags & PyPARSE_YIELD_IS_KEYWORD)
 116                 ps->p_generators = 1;
 117 #endif
 118
 119         for (;;) {
 120                 char *a, *b;
 121                 int type;
 122                 size_t len;
 123                 char *str;
 124
 125                 type = PyTokenizer_Get(tok, &a, &b);
 126                 if (type == ERRORTOKEN) {
 127                         err_ret->error = tok->done;
 128                         break;
 129                 }
 130                 if (type == ENDMARKER && started) {
 131                         type = NEWLINE; /* Add an extra newline */
 132                         started = 0;
 133                         /* Add the right number of dedent tokens,
 134                            except if a certain flag is given --
 135                            codeop.py uses this. */
 136                         if (tok->indent &&
 137                             !(flags & PyPARSE_DONT_IMPLY_DEDENT))
 138                         {
 139                                 tok->pendin = -tok->indent;
 140                                 tok->indent = 0;
 141                         }
 142                 }
 143                 else
 144                         started = 1;
 145                 len = b - a; /* XXX this may compute NULL - NULL */
 146                 str = (char *) PyObject_MALLOC(len + 1);
 147                 if (str == NULL) {
 148                         fprintf(stderr, "no mem for next token\n");
 149                         err_ret->error = E_NOMEM;
 150                         break;
 151                 }
 152                 if (len > 0)
 153                         strncpy(str, a, len);
 154                 str[len] = '\0';
 155
 156 #if 0 /* future keyword */
 157                 /* Warn about yield as NAME */
 158                 if (type == NAME && !ps->p_generators &&
 159                     len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
 160                         PySys_WriteStderr(yield_msg,
 161                                           err_ret->filename==NULL ?
 162                                           "<string>" : err_ret->filename,
 163                                           tok->lineno);
 164 #endif
 165
 166                 if ((err_ret->error =
 167                      PyParser_AddToken(ps, (int)type, str, tok->lineno,
 168                                        &(err_ret->expected))) != E_OK) {
 169                         if (err_ret->error != E_DONE)
 170                                 PyObject_FREE(str);
 171                         break;
 172                 }
 173         }
 174
 175         if (err_ret->error == E_DONE) {
 176                 n = ps->p_tree;
 177                 ps->p_tree = NULL;
 178         }
 179         else
 180                 n = NULL;
 181
 182         PyParser_Delete(ps);
 183
 184         if (n == NULL) {
 185                 if (tok->lineno <= 1 && tok->done == E_EOF)
 186                         err_ret->error = E_EOF;
 187                 err_ret->lineno = tok->lineno;
 188                 err_ret->offset = tok->cur - tok->buf;
 189                 if (tok->buf != NULL) {
 190                         size_t len = tok->inp - tok->buf;
 191                         err_ret->text = (char *) PyObject_MALLOC(len + 1);
 192                         if (err_ret->text != NULL) {
 193                                 if (len > 0)
 194                                         strncpy(err_ret->text, tok->buf, len);
 195                                 err_ret->text[len] = '\0';
 196                         }
 197                 }
 198         } else if (tok->encoding != NULL) {
 199                 node* r = PyNode_New(encoding_decl);
 200                 r->n_str = tok->encoding;
 201                 r->n_nchildren = 1;
 202                 r->n_child = n;
 203                 tok->encoding = NULL;
 204                 n = r;
 205         }
 206
 207         PyTokenizer_Free(tok);
 208
 209         return n;
 210 }
 211
 212 static void
 213 initerr(perrdetail *err_ret, const char *filename)
 214 {
 215         err_ret->error = E_OK;
 216         err_ret->filename = filename;
 217         err_ret->lineno = 0;
 218         err_ret->offset = 0;
 219         err_ret->text = NULL;
 220         err_ret->token = -1;
 221         err_ret->expected = -1;
 222 }