src/lj_lex.c

   1 /*
   2 ** Lexical analyzer.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 **
   5 ** Major portions taken verbatim or adapted from the Lua interpreter.
   6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7 */
   8
   9 #define lj_lex_c
  10 #define LUA_CORE
  11
  12 #include "lj_obj.h"
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_buf.h"
  16 #include "lj_str.h"
  17 #if LJ_HASFFI
  18 #include "lj_tab.h"
  19 #include "lj_ctype.h"
  20 #include "lj_cdata.h"
  21 #include "lualib.h"
  22 #endif
  23 #include "lj_state.h"
  24 #include "lj_lex.h"
  25 #include "lj_parse.h"
  26 #include "lj_char.h"
  27 #include "lj_strscan.h"
  28 #include "lj_strfmt.h"
  29
  30 /* Lua lexer token names. */
  31 static const char *const tokennames[] = {
  32 #define TKSTR1(name)            #name,
  33 #define TKSTR2(name, sym)       #sym,
  34 TKDEF(TKSTR1, TKSTR2)
  35 #undef TKSTR1
  36 #undef TKSTR2
  37   NULL
  38 };
  39
  40 /* -- Buffer handling ----------------------------------------------------- */
  41
  42 #define LEX_EOF                 (-1)
  43 #define lex_iseol(ls)           (ls->c == '\n' || ls->c == '\r')
  44
  45 /* Get more input from reader. */
  46 static LJ_NOINLINE LexChar lex_more(LexState *ls)
  47 {
  48   size_t sz;
  49   const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
  50   if (p == NULL || sz == 0) return LEX_EOF;
  51   if (sz >= LJ_MAX_BUF) {
  52     if (sz != ~(size_t)0) lj_err_mem(ls->L);
  53     sz = ~(uintptr_t)0 - (uintptr_t)p;
  54     if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1;
  55     ls->endmark = 1;
  56   }
  57   ls->pe = p + sz;
  58   ls->p = p + 1;
  59   return (LexChar)(uint8_t)p[0];
  60 }
  61
  62 /* Get next character. */
  63 static LJ_AINLINE LexChar lex_next(LexState *ls)
  64 {
  65   return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
  66 }
  67
  68 /* Save character. */
  69 static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
  70 {
  71   lj_buf_putb(&ls->sb, c);
  72 }
  73
  74 /* Save previous character and get next character. */
  75 static LJ_AINLINE LexChar lex_savenext(LexState *ls)
  76 {
  77   lex_save(ls, ls->c);
  78   return lex_next(ls);
  79 }
  80
  81 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
  82 static void lex_newline(LexState *ls)
  83 {
  84   LexChar old = ls->c;
  85   lj_assertLS(lex_iseol(ls), "bad usage");
  86   lex_next(ls);  /* Skip "\n" or "\r". */
  87   if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
  88   if (++ls->linenumber >= LJ_MAX_LINE)
  89     lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
  90 }
  91
  92 /* -- Scanner for terminals ----------------------------------------------- */
  93
  94 /* Parse a number literal. */
  95 static void lex_number(LexState *ls, TValue *tv)
  96 {
  97   StrScanFmt fmt;
  98   LexChar c, xp = 'e';
  99   lj_assertLS(lj_char_isdigit(ls->c), "bad usage");
 100   if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
 101     xp = 'p';
 102   while (lj_char_isident(ls->c) || ls->c == '.' ||
 103          ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
 104     c = ls->c;
 105     lex_savenext(ls);
 106   }
 107   lex_save(ls, '\0');
 108   fmt = lj_strscan_scan((const uint8_t *)ls->sb.b, sbuflen(&ls->sb)-1, tv,
 109           (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
 110           (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
 111   if (LJ_DUALNUM && fmt == STRSCAN_INT) {
 112     setitype(tv, LJ_TISNUM);
 113   } else if (fmt == STRSCAN_NUM) {
 114     /* Already in correct format. */
 115 #if LJ_HASFFI
 116   } else if (fmt != STRSCAN_ERROR) {
 117     lua_State *L = ls->L;
 118     GCcdata *cd;
 119     lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG,
 120                 "unexpected number format %d", fmt);
 121     ctype_loadffi(L);
 122     if (fmt == STRSCAN_IMAG) {
 123       cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
 124       ((double *)cdataptr(cd))[0] = 0;
 125       ((double *)cdataptr(cd))[1] = numV(tv);
 126     } else {
 127       cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
 128       *(uint64_t *)cdataptr(cd) = tv->u64;
 129     }
 130     lj_parse_keepcdata(ls, tv, cd);
 131 #endif
 132   } else {
 133     lj_assertLS(fmt == STRSCAN_ERROR,
 134                 "unexpected number format %d", fmt);
 135     lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
 136   }
 137 }
 138
 139 /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
 140 static int lex_skipeq(LexState *ls)
 141 {
 142   int count = 0;
 143   LexChar s = ls->c;
 144   lj_assertLS(s == '[' || s == ']', "bad usage");
 145   while (lex_savenext(ls) == '=' && count < 0x20000000)
 146     count++;
 147   return (ls->c == s) ? count : (-count) - 1;
 148 }
 149
 150 /* Parse a long string or long comment (tv set to NULL). */
 151 static void lex_longstring(LexState *ls, TValue *tv, int sep)
 152 {
 153   lex_savenext(ls);  /* Skip second '['. */
 154   if (lex_iseol(ls))  /* Skip initial newline. */
 155     lex_newline(ls);
 156   for (;;) {
 157     switch (ls->c) {
 158     case LEX_EOF:
 159       lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
 160       break;
 161     case ']':
 162       if (lex_skipeq(ls) == sep) {
 163         lex_savenext(ls);  /* Skip second ']'. */
 164         goto endloop;
 165       }
 166       break;
 167     case '\n':
 168     case '\r':
 169       lex_save(ls, '\n');
 170       lex_newline(ls);
 171       if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
 172       break;
 173     default:
 174       lex_savenext(ls);
 175       break;
 176     }
 177   } endloop:
 178   if (tv) {
 179     GCstr *str = lj_parse_keepstr(ls, ls->sb.b + (2 + (MSize)sep),
 180                                       sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
 181     setstrV(ls->L, tv, str);
 182   }
 183 }
 184
 185 /* Parse a string. */
 186 static void lex_string(LexState *ls, TValue *tv)
 187 {
 188   LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
 189   lex_savenext(ls);
 190   while (ls->c != delim) {
 191     switch (ls->c) {
 192     case LEX_EOF:
 193       lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
 194       continue;
 195     case '\n':
 196     case '\r':
 197       lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
 198       continue;
 199     case '\\': {
 200       LexChar c = lex_next(ls);  /* Skip the '\\'. */
 201       switch (c) {
 202       case 'a': c = '\a'; break;
 203       case 'b': c = '\b'; break;
 204       case 'f': c = '\f'; break;
 205       case 'n': c = '\n'; break;
 206       case 'r': c = '\r'; break;
 207       case 't': c = '\t'; break;
 208       case 'v': c = '\v'; break;
 209       case 'x':  /* Hexadecimal escape '\xXX'. */
 210         c = (lex_next(ls) & 15u) << 4;
 211         if (!lj_char_isdigit(ls->c)) {
 212           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 213           c += 9 << 4;
 214         }
 215         c += (lex_next(ls) & 15u);
 216         if (!lj_char_isdigit(ls->c)) {
 217           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 218           c += 9;
 219         }
 220         break;
 221       case 'u':  /* Unicode escape '\u{XX...}'. */
 222         if (lex_next(ls) != '{') goto err_xesc;
 223         lex_next(ls);
 224         c = 0;
 225         do {
 226           c = (c << 4) | (ls->c & 15u);
 227           if (!lj_char_isdigit(ls->c)) {
 228             if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 229             c += 9;
 230           }
 231           if (c >= 0x110000) goto err_xesc;  /* Out of Unicode range. */
 232         } while (lex_next(ls) != '}');
 233         if (c < 0x800) {
 234           if (c < 0x80) break;
 235           lex_save(ls, 0xc0 | (c >> 6));
 236         } else {
 237           if (c >= 0x10000) {
 238             lex_save(ls, 0xf0 | (c >> 18));
 239             lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
 240           } else {
 241             if (c >= 0xd800 && c < 0xe000) goto err_xesc;  /* No surrogates. */
 242             lex_save(ls, 0xe0 | (c >> 12));
 243           }
 244           lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
 245         }
 246         c = 0x80 | (c & 0x3f);
 247         break;
 248       case 'z':  /* Skip whitespace. */
 249         lex_next(ls);
 250         while (lj_char_isspace(ls->c))
 251           if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
 252         continue;
 253       case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
 254       case '\\': case '\"': case '\'': break;
 255       case LEX_EOF: continue;
 256       default:
 257         if (!lj_char_isdigit(c))
 258           goto err_xesc;
 259         c -= '0';  /* Decimal escape '\ddd'. */
 260         if (lj_char_isdigit(lex_next(ls))) {
 261           c = c*10 + (ls->c - '0');
 262           if (lj_char_isdigit(lex_next(ls))) {
 263             c = c*10 + (ls->c - '0');
 264             if (c > 255) {
 265             err_xesc:
 266               lj_lex_error(ls, TK_string, LJ_ERR_XESC);
 267             }
 268             lex_next(ls);
 269           }
 270         }
 271         lex_save(ls, c);
 272         continue;
 273       }
 274       lex_save(ls, c);
 275       lex_next(ls);
 276       continue;
 277       }
 278     default:
 279       lex_savenext(ls);
 280       break;
 281     }
 282   }
 283   lex_savenext(ls);  /* Skip trailing delimiter. */
 284   setstrV(ls->L, tv,
 285           lj_parse_keepstr(ls, ls->sb.b+1, sbuflen(&ls->sb)-2));
 286 }
 287
 288 /* -- Main lexical scanner ------------------------------------------------ */
 289
 290 /* Get next lexical token. */
 291 static LexToken lex_scan(LexState *ls, TValue *tv)
 292 {
 293   lj_buf_reset(&ls->sb);
 294   for (;;) {
 295     if (lj_char_isident(ls->c)) {
 296       GCstr *s;
 297       if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
 298         lex_number(ls, tv);
 299         return TK_number;
 300       }
 301       /* Identifier or reserved word. */
 302       do {
 303         lex_savenext(ls);
 304       } while (lj_char_isident(ls->c));
 305       s = lj_parse_keepstr(ls, ls->sb.b, sbuflen(&ls->sb));
 306       setstrV(ls->L, tv, s);
 307       if (s->reserved > 0)  /* Reserved word? */
 308         return TK_OFS + s->reserved;
 309       return TK_name;
 310     }
 311     switch (ls->c) {
 312     case '\n':
 313     case '\r':
 314       lex_newline(ls);
 315       continue;
 316     case ' ':
 317     case '\t':
 318     case '\v':
 319     case '\f':
 320       lex_next(ls);
 321       continue;
 322     case '-':
 323       lex_next(ls);
 324       if (ls->c != '-') return '-';
 325       lex_next(ls);
 326       if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
 327         int sep = lex_skipeq(ls);
 328         lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
 329         if (sep >= 0) {
 330           lex_longstring(ls, NULL, sep);
 331           lj_buf_reset(&ls->sb);
 332           continue;
 333         }
 334       }
 335       /* Short comment "--.*\n". */
 336       while (!lex_iseol(ls) && ls->c != LEX_EOF)
 337         lex_next(ls);
 338       continue;
 339     case '[': {
 340       int sep = lex_skipeq(ls);
 341       if (sep >= 0) {
 342         lex_longstring(ls, tv, sep);
 343         return TK_string;
 344       } else if (sep == -1) {
 345         return '[';
 346       } else {
 347         lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
 348         continue;
 349       }
 350       }
 351     case '=':
 352       lex_next(ls);
 353       if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
 354     case '<':
 355       lex_next(ls);
 356       if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
 357     case '>':
 358       lex_next(ls);
 359       if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
 360     case '~':
 361       lex_next(ls);
 362       if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
 363     case ':':
 364       lex_next(ls);
 365       if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
 366     case '"':
 367     case '\'':
 368       lex_string(ls, tv);
 369       return TK_string;
 370     case '.':
 371       if (lex_savenext(ls) == '.') {
 372         lex_next(ls);
 373         if (ls->c == '.') {
 374           lex_next(ls);
 375           return TK_dots;   /* ... */
 376         }
 377         return TK_concat;   /* .. */
 378       } else if (!lj_char_isdigit(ls->c)) {
 379         return '.';
 380       } else {
 381         lex_number(ls, tv);
 382         return TK_number;
 383       }
 384     case LEX_EOF:
 385       return TK_eof;
 386     default: {
 387       LexChar c = ls->c;
 388       lex_next(ls);
 389       return c;  /* Single-char tokens (+ - / ...). */
 390     }
 391     }
 392   }
 393 }
 394
 395 /* -- Lexer API ----------------------------------------------------------- */
 396
 397 /* Setup lexer state. */
 398 int lj_lex_setup(lua_State *L, LexState *ls)
 399 {
 400   int header = 0;
 401   ls->L = L;
 402   ls->fs = NULL;
 403   ls->pe = ls->p = NULL;
 404   ls->vstack = NULL;
 405   ls->sizevstack = 0;
 406   ls->vtop = 0;
 407   ls->bcstack = NULL;
 408   ls->sizebcstack = 0;
 409   ls->tok = 0;
 410   ls->lookahead = TK_eof;  /* No look-ahead token. */
 411   ls->linenumber = 1;
 412   ls->lastline = 1;
 413   ls->endmark = 0;
 414   ls->fr2 = LJ_FR2;  /* Generate native bytecode by default. */
 415   lex_next(ls);  /* Read-ahead first char. */
 416   if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
 417       (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
 418     ls->p += 2;
 419     lex_next(ls);
 420     header = 1;
 421   }
 422   if (ls->c == '#') {  /* Skip POSIX #! header line. */
 423     do {
 424       lex_next(ls);
 425       if (ls->c == LEX_EOF) return 0;
 426     } while (!lex_iseol(ls));
 427     lex_newline(ls);
 428     header = 1;
 429   }
 430   if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
 431     if (header) {
 432       /*
 433       ** Loading bytecode with an extra header is disabled for security
 434       ** reasons. This may circumvent the usual check for bytecode vs.
 435       ** Lua code by looking at the first char. Since this is a potential
 436       ** security violation no attempt is made to echo the chunkname either.
 437       */
 438       setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
 439       lj_err_throw(L, LUA_ERRSYNTAX);
 440     }
 441     return 1;
 442   }
 443   return 0;
 444 }
 445
 446 /* Cleanup lexer state. */
 447 void lj_lex_cleanup(lua_State *L, LexState *ls)
 448 {
 449   global_State *g = G(L);
 450   lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
 451   lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
 452   lj_buf_free(g, &ls->sb);
 453 }
 454
 455 /* Return next lexical token. */
 456 void lj_lex_next(LexState *ls)
 457 {
 458   ls->lastline = ls->linenumber;
 459   if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
 460     ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
 461   } else {  /* Otherwise return lookahead token. */
 462     ls->tok = ls->lookahead;
 463     ls->lookahead = TK_eof;
 464     ls->tokval = ls->lookaheadval;
 465   }
 466 }
 467
 468 /* Look ahead for the next token. */
 469 LexToken lj_lex_lookahead(LexState *ls)
 470 {
 471   lj_assertLS(ls->lookahead == TK_eof, "double lookahead");
 472   ls->lookahead = lex_scan(ls, &ls->lookaheadval);
 473   return ls->lookahead;
 474 }
 475
 476 /* Convert token to string. */
 477 const char *lj_lex_token2str(LexState *ls, LexToken tok)
 478 {
 479   if (tok > TK_OFS)
 480     return tokennames[tok-TK_OFS-1];
 481   else if (!lj_char_iscntrl(tok))
 482     return lj_strfmt_pushf(ls->L, "%c", tok);
 483   else
 484     return lj_strfmt_pushf(ls->L, "char(%d)", tok);
 485 }
 486
 487 /* Lexer error. */
 488 void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
 489 {
 490   const char *tokstr;
 491   va_list argp;
 492   if (tok == 0) {
 493     tokstr = NULL;
 494   } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
 495     lex_save(ls, '\0');
 496     tokstr = ls->sb.b;
 497   } else {
 498     tokstr = lj_lex_token2str(ls, tok);
 499   }
 500   va_start(argp, em);
 501   lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
 502   va_end(argp);
 503 }
 504
 505 /* Initialize strings for reserved words. */
 506 void lj_lex_init(lua_State *L)
 507 {
 508   uint32_t i;
 509   for (i = 0; i < TK_RESERVED; i++) {
 510     GCstr *s = lj_str_newz(L, tokennames[i]);
 511     fixstring(s);  /* Reserved words are never collected. */
 512     s->reserved = (uint8_t)(i+1);
 513   }
 514 }
 515