src/lua/llex.cpp

   1 /*
   2 ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $
   3 ** Lexical Analyzer
   4 ** See Copyright Notice in lua.h
   5 */
   6
   7
   8 #include <ctype.h>
   9 #include <locale.h>
  10 #include <string.h>
  11
  12 #define llex_c
  13 #define LUA_CORE
  14
  15 #include "lua.h"
  16
  17 #include "ldo.h"
  18 #include "llex.h"
  19 #include "lobject.h"
  20 #include "lparser.h"
  21 #include "lstate.h"
  22 #include "lstring.h"
  23 #include "ltable.h"
  24 #include "lzio.h"
  25
  26
  27
  28 #define next(ls) (ls->current = zgetc(ls->z))
  29
  30
  31
  32
  33 #define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
  34
  35
  36 /* ORDER RESERVED */
  37 const char *const luaX_tokens [] = {
  38     "and", "break", "do", "else", "elseif",
  39     "end", "false", "for", "function", "if",
  40     "in", "local", "nil", "not", "or", "repeat",
  41     "return", "then", "true", "until", "while",
  42     "..", "...", "==", ">=", "<=", "~=",
  43     "<number>", "<name>", "<string>", "<eof>",
  44     NULL
  45 };
  46
  47
  48 #define save_and_next(ls) (save(ls, ls->current), next(ls))
  49
  50
  51 static void save (LexState *ls, int c) {
  52   Mbuffer *b = ls->buff;
  53   if (b->n + 1 > b->buffsize) {
  54     size_t newsize;
  55     if (b->buffsize >= MAX_SIZET/2)
  56       luaX_lexerror(ls, "lexical element too long", 0);
  57     newsize = b->buffsize * 2;
  58     luaZ_resizebuffer(ls->L, b, newsize);
  59   }
  60   b->buffer[b->n++] = cast(char, c);
  61 }
  62
  63
  64 void luaX_init (lua_State *L) {
  65   int i;
  66   for (i=0; i<NUM_RESERVED; i++) {
  67     TString *ts = luaS_new(L, luaX_tokens[i]);
  68     luaS_fix(ts);  /* reserved words are never collected */
  69     lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
  70     ts->tsv.reserved = cast_byte(i+1);  /* reserved word */
  71   }
  72 }
  73
  74
  75 #define MAXSRC          80
  76
  77
  78 const char *luaX_token2str (LexState *ls, int token) {
  79   if (token < FIRST_RESERVED) {
  80     lua_assert(token == cast(unsigned char, token));
  81     return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  82                               luaO_pushfstring(ls->L, "%c", token);
  83   }
  84   else
  85     return luaX_tokens[token-FIRST_RESERVED];
  86 }
  87
  88
  89 static const char *txtToken (LexState *ls, int token) {
  90   switch (token) {
  91     case TK_NAME:
  92     case TK_STRING:
  93     case TK_NUMBER:
  94       save(ls, '\0');
  95       return luaZ_buffer(ls->buff);
  96     default:
  97       return luaX_token2str(ls, token);
  98   }
  99 }
 100
 101
 102 void luaX_lexerror (LexState *ls, const char *msg, int token) {
 103   char buff[MAXSRC];
 104   luaO_chunkid(buff, getstr(ls->source), MAXSRC);
 105   msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
 106   if (token)
 107     luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
 108   luaD_throw(ls->L, LUA_ERRSYNTAX);
 109 }
 110
 111
 112 void luaX_syntaxerror (LexState *ls, const char *msg) {
 113   luaX_lexerror(ls, msg, ls->t.token);
 114 }
 115
 116
 117 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
 118   lua_State *L = ls->L;
 119   TString *ts = luaS_newlstr(L, str, l);
 120   TValue *o = luaH_setstr(L, ls->fs->h, ts);  /* entry for `str' */
 121   if (ttisnil(o))
 122     setbvalue(o, 1);  /* make sure `str' will not be collected */
 123   return ts;
 124 }
 125
 126
 127 static void inclinenumber (LexState *ls) {
 128   int old = ls->current;
 129   lua_assert(currIsNewline(ls));
 130   next(ls);  /* skip `\n' or `\r' */
 131   if (currIsNewline(ls) && ls->current != old)
 132     next(ls);  /* skip `\n\r' or `\r\n' */
 133   if (++ls->linenumber >= MAX_INT)
 134     luaX_syntaxerror(ls, "chunk has too many lines");
 135 }
 136
 137
 138 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
 139   ls->decpoint = '.';
 140   ls->L = L;
 141   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
 142   ls->z = z;
 143   ls->fs = NULL;
 144   ls->linenumber = 1;
 145   ls->lastline = 1;
 146   ls->source = source;
 147   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
 148   next(ls);  /* read first char */
 149 }
 150
 151
 152
 153 /*
 154 ** =======================================================
 155 ** LEXICAL ANALYZER
 156 ** =======================================================
 157 */
 158
 159
 160
 161 static int check_next (LexState *ls, const char *set) {
 162   if (!strchr(set, ls->current))
 163     return 0;
 164   save_and_next(ls);
 165   return 1;
 166 }
 167
 168
 169 static void buffreplace (LexState *ls, char from, char to) {
 170   size_t n = luaZ_bufflen(ls->buff);
 171   char *p = luaZ_buffer(ls->buff);
 172   while (n--)
 173     if (p[n] == from) p[n] = to;
 174 }
 175
 176
 177 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
 178   /* format error: try to update decimal point separator */
 179   struct lconv *cv = localeconv();
 180   char old = ls->decpoint;
 181   ls->decpoint = (cv ? cv->decimal_point[0] : '.');
 182   buffreplace(ls, old, ls->decpoint);  /* try updated decimal separator */
 183   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
 184     /* format error with correct decimal point: no more options */
 185     buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
 186     luaX_lexerror(ls, "malformed number", TK_NUMBER);
 187   }
 188 }
 189
 190
 191 /* LUA_NUMBER */
 192 static void read_numeral (LexState *ls, SemInfo *seminfo) {
 193   lua_assert(isdigit(ls->current));
 194   do {
 195     save_and_next(ls);
 196   } while (isdigit(ls->current) || ls->current == '.');
 197   if (check_next(ls, "Ee"))  /* `E'? */
 198     check_next(ls, "+-");  /* optional exponent sign */
 199   while (isalnum(ls->current) || ls->current == '_')
 200     save_and_next(ls);
 201   save(ls, '\0');
 202   buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
 203   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r))  /* format error? */
 204     trydecpoint(ls, seminfo); /* try to update decimal point separator */
 205 }
 206
 207
 208 static int skip_sep (LexState *ls) {
 209   int count = 0;
 210   int s = ls->current;
 211   lua_assert(s == '[' || s == ']');
 212   save_and_next(ls);
 213   while (ls->current == '=') {
 214     save_and_next(ls);
 215     count++;
 216   }
 217   return (ls->current == s) ? count : (-count) - 1;
 218 }
 219
 220
 221 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
 222   int cont = 0;
 223   (void)(cont);  /* avoid warnings when `cont' is not used */
 224   save_and_next(ls);  /* skip 2nd `[' */
 225   if (currIsNewline(ls))  /* string starts with a newline? */
 226     inclinenumber(ls);  /* skip it */
 227   for (;;) {
 228     switch (ls->current) {
 229       case EOZ:
 230         luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
 231                                    "unfinished long comment", TK_EOS);
 232         break;  /* to avoid warnings */
 233 #if defined(LUA_COMPAT_LSTR)
 234       case '[': {
 235         if (skip_sep(ls) == sep) {
 236           save_and_next(ls);  /* skip 2nd `[' */
 237           cont++;
 238 #if LUA_COMPAT_LSTR == 1
 239           if (sep == 0)
 240             luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
 241 #endif
 242         }
 243         break;
 244       }
 245 #endif
 246       case ']': {
 247         if (skip_sep(ls) == sep) {
 248           save_and_next(ls);  /* skip 2nd `]' */
 249 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
 250           cont--;
 251           if (sep == 0 && cont >= 0) break;
 252 #endif
 253           goto endloop;
 254         }
 255         break;
 256       }
 257       case '\n':
 258       case '\r': {
 259         save(ls, '\n');
 260         inclinenumber(ls);
 261         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
 262         break;
 263       }
 264       default: {
 265         if (seminfo) save_and_next(ls);
 266         else next(ls);
 267       }
 268     }
 269   } endloop:
 270   if (seminfo)
 271     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
 272                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
 273 }
 274
 275
 276 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
 277   save_and_next(ls);
 278   while (ls->current != del) {
 279     switch (ls->current) {
 280       case EOZ:
 281         luaX_lexerror(ls, "unfinished string", TK_EOS);
 282         continue;  /* to avoid warnings */
 283       case '\n':
 284       case '\r':
 285         luaX_lexerror(ls, "unfinished string", TK_STRING);
 286         continue;  /* to avoid warnings */
 287       case '\\': {
 288         int c;
 289         next(ls);  /* do not save the `\' */
 290         switch (ls->current) {
 291           case 'a': c = '\a'; break;
 292           case 'b': c = '\b'; break;
 293           case 'f': c = '\f'; break;
 294           case 'n': c = '\n'; break;
 295           case 'r': c = '\r'; break;
 296           case 't': c = '\t'; break;
 297           case 'v': c = '\v'; break;
 298           case '\n':  /* go through */
 299           case '\r': save(ls, '\n'); inclinenumber(ls); continue;
 300           case EOZ: continue;  /* will raise an error next loop */
 301           default: {
 302             if (!isdigit(ls->current))
 303               save_and_next(ls);  /* handles \\, \", \', and \? */
 304             else {  /* \xxx */
 305               int i = 0;
 306               c = 0;
 307               do {
 308                 c = 10*c + (ls->current-'0');
 309                 next(ls);
 310               } while (++i<3 && isdigit(ls->current));
 311               if (c > UCHAR_MAX)
 312                 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
 313               save(ls, c);
 314             }
 315             continue;
 316           }
 317         }
 318         save(ls, c);
 319         next(ls);
 320         continue;
 321       }
 322       default:
 323         save_and_next(ls);
 324     }
 325   }
 326   save_and_next(ls);  /* skip delimiter */
 327   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
 328                                    luaZ_bufflen(ls->buff) - 2);
 329 }
 330
 331
 332 static int llex (LexState *ls, SemInfo *seminfo) {
 333   luaZ_resetbuffer(ls->buff);
 334   for (;;) {
 335     switch (ls->current) {
 336       case '\n':
 337       case '\r': {
 338         inclinenumber(ls);
 339         continue;
 340       }
 341       case '-': {
 342         next(ls);
 343         if (ls->current != '-') return '-';
 344         /* else is a comment */
 345         next(ls);
 346         if (ls->current == '[') {
 347           int sep = skip_sep(ls);
 348           luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
 349           if (sep >= 0) {
 350             read_long_string(ls, NULL, sep);  /* long comment */
 351             luaZ_resetbuffer(ls->buff);
 352             continue;
 353           }
 354         }
 355         /* else short comment */
 356         while (!currIsNewline(ls) && ls->current != EOZ)
 357           next(ls);
 358         continue;
 359       }
 360       case '[': {
 361         int sep = skip_sep(ls);
 362         if (sep >= 0) {
 363           read_long_string(ls, seminfo, sep);
 364           return TK_STRING;
 365         }
 366         else if (sep == -1) return '[';
 367         else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
 368       }
 369       case '=': {
 370         next(ls);
 371         if (ls->current != '=') return '=';
 372         else { next(ls); return TK_EQ; }
 373       }
 374       case '<': {
 375         next(ls);
 376         if (ls->current != '=') return '<';
 377         else { next(ls); return TK_LE; }
 378       }
 379       case '>': {
 380         next(ls);
 381         if (ls->current != '=') return '>';
 382         else { next(ls); return TK_GE; }
 383       }
 384       case '~': {
 385         next(ls);
 386         if (ls->current != '=') return '~';
 387         else { next(ls); return TK_NE; }
 388       }
 389       case '"':
 390       case '\'': {
 391         read_string(ls, ls->current, seminfo);
 392         return TK_STRING;
 393       }
 394       case '.': {
 395         save_and_next(ls);
 396         if (check_next(ls, ".")) {
 397           if (check_next(ls, "."))
 398             return TK_DOTS;   /* ... */
 399           else return TK_CONCAT;   /* .. */
 400         }
 401         else if (!isdigit(ls->current)) return '.';
 402         else {
 403           read_numeral(ls, seminfo);
 404           return TK_NUMBER;
 405         }
 406       }
 407       case EOZ: {
 408         return TK_EOS;
 409       }
 410       default: {
 411         if (isspace(ls->current)) {
 412           lua_assert(!currIsNewline(ls));
 413           next(ls);
 414           continue;
 415         }
 416         else if (isdigit(ls->current)) {
 417           read_numeral(ls, seminfo);
 418           return TK_NUMBER;
 419         }
 420         else if (isalpha(ls->current) || ls->current == '_') {
 421           /* identifier or reserved word */
 422           TString *ts;
 423           do {
 424             save_and_next(ls);
 425           } while (isalnum(ls->current) || ls->current == '_');
 426           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
 427                                   luaZ_bufflen(ls->buff));
 428           if (ts->tsv.reserved > 0)  /* reserved word? */
 429             return ts->tsv.reserved - 1 + FIRST_RESERVED;
 430           else {
 431             seminfo->ts = ts;
 432             return TK_NAME;
 433           }
 434         }
 435         else {
 436           int c = ls->current;
 437           next(ls);
 438           return c;  /* single-char tokens (+ - / ...) */
 439         }
 440       }
 441     }
 442   }
 443 }
 444
 445
 446 void luaX_next (LexState *ls) {
 447   ls->lastline = ls->linenumber;
 448   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
 449     ls->t = ls->lookahead;  /* use this one */
 450     ls->lookahead.token = TK_EOS;  /* and discharge it */
 451   }
 452   else
 453     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
 454 }
 455
 456
 457 void luaX_lookahead (LexState *ls) {
 458   lua_assert(ls->lookahead.token == TK_EOS);
 459   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
 460 }
 461