src/lj_lex.c

   1 /*
   2 ** Lexical analyzer.
   3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
   4 **
   5 ** Major portions taken verbatim or adapted from the Lua interpreter.
   6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7 */
   8
   9 #define lj_lex_c
  10 #define LUA_CORE
  11
  12 #include "lj_obj.h"
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_str.h"
  16 #if LJ_HASFFI
  17 #include "lj_tab.h"
  18 #include "lj_ctype.h"
  19 #include "lj_cdata.h"
  20 #include "lualib.h"
  21 #endif
  22 #include "lj_state.h"
  23 #include "lj_lex.h"
  24 #include "lj_parse.h"
  25 #include "lj_char.h"
  26
  27 /* Lua lexer token names. */
  28 static const char *const tokennames[] = {
  29 #define TKSTR1(name)            #name,
  30 #define TKSTR2(name, sym)       #sym,
  31 TKDEF(TKSTR1, TKSTR2)
  32 #undef TKSTR1
  33 #undef TKSTR2
  34   NULL
  35 };
  36
  37 /* -- Buffer handling ----------------------------------------------------- */
  38
  39 #define char2int(c)             ((int)(uint8_t)(c))
  40 #define next(ls) \
  41   (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
  42 #define save_and_next(ls)       (save(ls, ls->current), next(ls))
  43 #define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
  44 #define END_OF_STREAM           (-1)
  45
  46 static int fillbuf(LexState *ls)
  47 {
  48   size_t sz;
  49   const char *buf = ls->rfunc(ls->L, ls->rdata, &sz);
  50   if (buf == NULL || sz == 0) return END_OF_STREAM;
  51   ls->n = (MSize)sz - 1;
  52   ls->p = buf;
  53   return char2int(*(ls->p++));
  54 }
  55
  56 static LJ_NOINLINE void save_grow(LexState *ls, int c)
  57 {
  58   MSize newsize;
  59   if (ls->sb.sz >= LJ_MAX_STR/2)
  60     lj_lex_error(ls, 0, LJ_ERR_XELEM);
  61   newsize = ls->sb.sz * 2;
  62   lj_str_resizebuf(ls->L, &ls->sb, newsize);
  63   ls->sb.buf[ls->sb.n++] = (char)c;
  64 }
  65
  66 static LJ_AINLINE void save(LexState *ls, int c)
  67 {
  68   if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz))
  69     save_grow(ls, c);
  70   else
  71     ls->sb.buf[ls->sb.n++] = (char)c;
  72 }
  73
  74 static void inclinenumber(LexState *ls)
  75 {
  76   int old = ls->current;
  77   lua_assert(currIsNewline(ls));
  78   next(ls);  /* skip `\n' or `\r' */
  79   if (currIsNewline(ls) && ls->current != old)
  80     next(ls);  /* skip `\n\r' or `\r\n' */
  81   if (++ls->linenumber >= LJ_MAX_LINE)
  82     lj_lex_error(ls, ls->token, LJ_ERR_XLINES);
  83 }
  84
  85 /* -- Scanner for terminals ----------------------------------------------- */
  86
  87 #if LJ_HASFFI
  88 /* Load FFI library on-demand. Needed if we create cdata objects. */
  89 static void lex_loadffi(lua_State *L)
  90 {
  91   ptrdiff_t oldtop = savestack(L, L->top);
  92   luaopen_ffi(L);
  93   L->top = restorestack(L, oldtop);
  94 }
  95
  96 /* Parse 64 bit integer. */
  97 static int lex_number64(LexState *ls, TValue *tv)
  98 {
  99   uint64_t n = 0;
 100   uint8_t *p = (uint8_t *)ls->sb.buf;
 101   CTypeID id = CTID_INT64;
 102   GCcdata *cd;
 103   int numl = 0;
 104   if (p[0] == '0' && (p[1] & ~0x20) == 'X') {  /* Hexadecimal. */
 105     p += 2;
 106     if (!lj_char_isxdigit(*p)) return 0;
 107     do {
 108       n = n*16 + (*p & 15);
 109       if (!lj_char_isdigit(*p)) n += 9;
 110       p++;
 111     } while (lj_char_isxdigit(*p));
 112   } else {  /* Decimal. */
 113     if (!lj_char_isdigit(*p)) return 0;
 114     do {
 115       n = n*10 + (*p - '0');
 116       p++;
 117     } while (lj_char_isdigit(*p));
 118   }
 119   for (;;) {  /* Parse suffixes. */
 120     if ((*p & ~0x20) == 'U')
 121       id = CTID_UINT64;
 122     else if ((*p & ~0x20) == 'L')
 123       numl++;
 124     else
 125       break;
 126     p++;
 127   }
 128   if (numl != 2 || *p != '\0') return 0;
 129   /* Return cdata holding a 64 bit integer. */
 130   cd = lj_cdata_new_(ls->L, id, 8);
 131   *(uint64_t *)cdataptr(cd) = n;
 132   lj_parse_keepcdata(ls, tv, cd);
 133   return 1;  /* Ok. */
 134 }
 135 #endif
 136
 137 /* Parse a number literal. */
 138 static void lex_number(LexState *ls, TValue *tv)
 139 {
 140   int c, xp = 'E';
 141   lua_assert(lj_char_isdigit(ls->current));
 142   if ((c = ls->current) == '0') {
 143     save_and_next(ls);
 144     if ((ls->current & ~0x20) == 'X') xp = 'P';
 145   }
 146   while (lj_char_isident(ls->current) || ls->current == '.' ||
 147          ((ls->current == '-' || ls->current == '+') && (c & ~0x20) == xp)) {
 148     c = ls->current;
 149     save_and_next(ls);
 150   }
 151 #if LJ_HASFFI
 152   c &= ~0x20;
 153   if ((c == 'I' || c == 'L' || c == 'U') && !ctype_ctsG(G(ls->L)))
 154     lex_loadffi(ls->L);
 155   if (c == 'I')  /* Parse imaginary part of complex number. */
 156     ls->sb.n--;
 157 #endif
 158   save(ls, '\0');
 159 #if LJ_HASFFI
 160   if ((c == 'L' || c == 'U') && lex_number64(ls, tv)) {  /* Parse 64 bit int. */
 161     return;
 162   } else
 163 #endif
 164   if (lj_str_numconv(ls->sb.buf, tv)) {
 165 #if LJ_HASFFI
 166     if (c == 'I') {  /* Return cdata holding a complex number. */
 167       GCcdata *cd = lj_cdata_new_(ls->L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
 168       ((double *)cdataptr(cd))[0] = 0;
 169       ((double *)cdataptr(cd))[1] = numberVnum(tv);
 170       lj_parse_keepcdata(ls, tv, cd);
 171     }
 172 #endif
 173     if (LJ_DUALNUM && tvisnum(tv)) {
 174       int32_t k = lj_num2int(numV(tv));
 175       if ((lua_Number)k == numV(tv))  /* -0 cannot end up here. */
 176         setintV(tv, k);
 177     }
 178     return;
 179   }
 180   lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
 181 }
 182
 183 static int skip_sep(LexState *ls)
 184 {
 185   int count = 0;
 186   int s = ls->current;
 187   lua_assert(s == '[' || s == ']');
 188   save_and_next(ls);
 189   while (ls->current == '=') {
 190     save_and_next(ls);
 191     count++;
 192   }
 193   return (ls->current == s) ? count : (-count) - 1;
 194 }
 195
 196 static void read_long_string(LexState *ls, TValue *tv, int sep)
 197 {
 198   save_and_next(ls);  /* skip 2nd `[' */
 199   if (currIsNewline(ls))  /* string starts with a newline? */
 200     inclinenumber(ls);  /* skip it */
 201   for (;;) {
 202     switch (ls->current) {
 203     case END_OF_STREAM:
 204       lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
 205       break;
 206     case ']':
 207       if (skip_sep(ls) == sep) {
 208         save_and_next(ls);  /* skip 2nd `]' */
 209         goto endloop;
 210       }
 211       break;
 212     case '\n':
 213     case '\r':
 214       save(ls, '\n');
 215       inclinenumber(ls);
 216       if (!tv) lj_str_resetbuf(&ls->sb);  /* avoid wasting space */
 217       break;
 218     default:
 219       if (tv) save_and_next(ls);
 220       else next(ls);
 221       break;
 222     }
 223   } endloop:
 224   if (tv) {
 225     GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep),
 226                                       ls->sb.n - 2*(2 + (MSize)sep));
 227     setstrV(ls->L, tv, str);
 228   }
 229 }
 230
 231 static void read_string(LexState *ls, int delim, TValue *tv)
 232 {
 233   save_and_next(ls);
 234   while (ls->current != delim) {
 235     switch (ls->current) {
 236     case END_OF_STREAM:
 237       lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
 238       continue;
 239     case '\n':
 240     case '\r':
 241       lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
 242       continue;
 243     case '\\': {
 244       int c = next(ls);  /* Skip the '\\'. */
 245       switch (c) {
 246       case 'a': c = '\a'; break;
 247       case 'b': c = '\b'; break;
 248       case 'f': c = '\f'; break;
 249       case 'n': c = '\n'; break;
 250       case 'r': c = '\r'; break;
 251       case 't': c = '\t'; break;
 252       case 'v': c = '\v'; break;
 253       case 'x':  /* Hexadecimal escape '\xXX'. */
 254         c = (next(ls) & 15u) << 4;
 255         if (!lj_char_isdigit(ls->current)) {
 256           if (!lj_char_isxdigit(ls->current)) goto err_xesc;
 257           c += 9 << 4;
 258         }
 259         c += (next(ls) & 15u);
 260         if (!lj_char_isdigit(ls->current)) {
 261           if (!lj_char_isxdigit(ls->current)) goto err_xesc;
 262           c += 9;
 263         }
 264         break;
 265       case 'z':  /* Skip whitespace. */
 266         next(ls);
 267         while (lj_char_isspace(ls->current))
 268           if (currIsNewline(ls)) inclinenumber(ls); else next(ls);
 269         continue;
 270       case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue;
 271       case '\\': case '\"': case '\'': break;
 272       case END_OF_STREAM: continue;
 273       default:
 274         if (!lj_char_isdigit(c))
 275           goto err_xesc;
 276         c -= '0';  /* Decimal escape '\ddd'. */
 277         if (lj_char_isdigit(next(ls))) {
 278           c = c*10 + (ls->current - '0');
 279           if (lj_char_isdigit(next(ls))) {
 280             c = c*10 + (ls->current - '0');
 281             if (c > 255) {
 282             err_xesc:
 283               lj_lex_error(ls, TK_string, LJ_ERR_XESC);
 284             }
 285             next(ls);
 286           }
 287         }
 288         save(ls, c);
 289         continue;
 290       }
 291       save(ls, c);
 292       next(ls);
 293       continue;
 294       }
 295     default:
 296       save_and_next(ls);
 297       break;
 298     }
 299   }
 300   save_and_next(ls);  /* skip delimiter */
 301   setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2));
 302 }
 303
 304 /* -- Main lexical scanner ------------------------------------------------ */
 305
 306 static int llex(LexState *ls, TValue *tv)
 307 {
 308   lj_str_resetbuf(&ls->sb);
 309   for (;;) {
 310     if (lj_char_isident(ls->current)) {
 311       GCstr *s;
 312       if (lj_char_isdigit(ls->current)) {  /* Numeric literal. */
 313         lex_number(ls, tv);
 314         return TK_number;
 315       }
 316       /* Identifier or reserved word. */
 317       do {
 318         save_and_next(ls);
 319       } while (lj_char_isident(ls->current));
 320       s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n);
 321       if (s->reserved > 0)  /* Reserved word? */
 322         return TK_OFS + s->reserved;
 323       setstrV(ls->L, tv, s);
 324       return TK_name;
 325     }
 326     switch (ls->current) {
 327     case '\n':
 328     case '\r':
 329       inclinenumber(ls);
 330       continue;
 331     case ' ':
 332     case '\t':
 333     case '\v':
 334     case '\f':
 335       next(ls);
 336       continue;
 337     case '-':
 338       next(ls);
 339       if (ls->current != '-') return '-';
 340       /* else is a comment */
 341       next(ls);
 342       if (ls->current == '[') {
 343         int sep = skip_sep(ls);
 344         lj_str_resetbuf(&ls->sb);  /* `skip_sep' may dirty the buffer */
 345         if (sep >= 0) {
 346           read_long_string(ls, NULL, sep);  /* long comment */
 347           lj_str_resetbuf(&ls->sb);
 348           continue;
 349         }
 350       }
 351       /* else short comment */
 352       while (!currIsNewline(ls) && ls->current != END_OF_STREAM)
 353         next(ls);
 354       continue;
 355     case '[': {
 356       int sep = skip_sep(ls);
 357       if (sep >= 0) {
 358         read_long_string(ls, tv, sep);
 359         return TK_string;
 360       } else if (sep == -1) {
 361         return '[';
 362       } else {
 363         lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
 364         continue;
 365       }
 366       }
 367     case '=':
 368       next(ls);
 369       if (ls->current != '=') return '='; else { next(ls); return TK_eq; }
 370     case '<':
 371       next(ls);
 372       if (ls->current != '=') return '<'; else { next(ls); return TK_le; }
 373     case '>':
 374       next(ls);
 375       if (ls->current != '=') return '>'; else { next(ls); return TK_ge; }
 376     case '~':
 377       next(ls);
 378       if (ls->current != '=') return '~'; else { next(ls); return TK_ne; }
 379     case '"':
 380     case '\'':
 381       read_string(ls, ls->current, tv);
 382       return TK_string;
 383     case '.':
 384       save_and_next(ls);
 385       if (ls->current == '.') {
 386         next(ls);
 387         if (ls->current == '.') {
 388           next(ls);
 389           return TK_dots;   /* ... */
 390         }
 391         return TK_concat;   /* .. */
 392       } else if (!lj_char_isdigit(ls->current)) {
 393         return '.';
 394       } else {
 395         lex_number(ls, tv);
 396         return TK_number;
 397       }
 398     case END_OF_STREAM:
 399       return TK_eof;
 400     default: {
 401       int c = ls->current;
 402       next(ls);
 403       return c;  /* Single-char tokens (+ - / ...). */
 404     }
 405     }
 406   }
 407 }
 408
 409 /* -- Lexer API ----------------------------------------------------------- */
 410
 411 /* Setup lexer state. */
 412 int lj_lex_setup(lua_State *L, LexState *ls)
 413 {
 414   ls->L = L;
 415   ls->fs = NULL;
 416   ls->n = 0;
 417   ls->p = NULL;
 418   ls->vstack = NULL;
 419   ls->sizevstack = 0;
 420   ls->vtop = 0;
 421   ls->bcstack = NULL;
 422   ls->sizebcstack = 0;
 423   ls->lookahead = TK_eof;  /* No look-ahead token. */
 424   ls->linenumber = 1;
 425   ls->lastline = 1;
 426   lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF);
 427   next(ls);  /* Read-ahead first char. */
 428   if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb &&
 429       char2int(ls->p[1]) == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
 430     ls->n -= 2;
 431     ls->p += 2;
 432     next(ls);
 433   }
 434   if (ls->current == '#') {  /* Skip POSIX #! header line. */
 435     do {
 436       next(ls);
 437       if (ls->current == END_OF_STREAM) return 0;
 438     } while (!currIsNewline(ls));
 439     inclinenumber(ls);
 440   }
 441   return (ls->current == LUA_SIGNATURE[0]);  /* Bytecode dump? */
 442 }
 443
 444 /* Cleanup lexer state. */
 445 void lj_lex_cleanup(lua_State *L, LexState *ls)
 446 {
 447   global_State *g = G(L);
 448   lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
 449   lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
 450   lj_str_freebuf(g, &ls->sb);
 451 }
 452
 453 void lj_lex_next(LexState *ls)
 454 {
 455   ls->lastline = ls->linenumber;
 456   if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
 457     ls->token = llex(ls, &ls->tokenval);  /* Get next token. */
 458   } else {  /* Otherwise return lookahead token. */
 459     ls->token = ls->lookahead;
 460     ls->lookahead = TK_eof;
 461     ls->tokenval = ls->lookaheadval;
 462   }
 463 }
 464
 465 LexToken lj_lex_lookahead(LexState *ls)
 466 {
 467   lua_assert(ls->lookahead == TK_eof);
 468   ls->lookahead = llex(ls, &ls->lookaheadval);
 469   return ls->lookahead;
 470 }
 471
 472 const char *lj_lex_token2str(LexState *ls, LexToken token)
 473 {
 474   if (token > TK_OFS)
 475     return tokennames[token-TK_OFS-1];
 476   else if (!lj_char_iscntrl(token))
 477     return lj_str_pushf(ls->L, "%c", token);
 478   else
 479     return lj_str_pushf(ls->L, "char(%d)", token);
 480 }
 481
 482 void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...)
 483 {
 484   const char *tok;
 485   va_list argp;
 486   if (token == 0) {
 487     tok = NULL;
 488   } else if (token == TK_name || token == TK_string || token == TK_number) {
 489     save(ls, '\0');
 490     tok = ls->sb.buf;
 491   } else {
 492     tok = lj_lex_token2str(ls, token);
 493   }
 494   va_start(argp, em);
 495   lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp);
 496   va_end(argp);
 497 }
 498
 499 void lj_lex_init(lua_State *L)
 500 {
 501   uint32_t i;
 502   for (i = 0; i < TK_RESERVED; i++) {
 503     GCstr *s = lj_str_newz(L, tokennames[i]);
 504     fixstring(s);  /* Reserved words are never collected. */
 505     s->reserved = (uint8_t)(i+1);
 506   }
 507 }
 508