3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
5 ** Major portions taken verbatim or adapted from the Lua interpreter.
6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
27 #include "lj_strscan.h"
28 #include "lj_strfmt.h"
30 /* Lua lexer token names. */
31 static const char *const tokennames
[] = {
32 #define TKSTR1(name) #name,
33 #define TKSTR2(name, sym) #sym,
40 /* -- Buffer handling ----------------------------------------------------- */
43 #define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
45 /* Get more input from reader. */
46 static LJ_NOINLINE LexChar
lex_more(LexState
*ls
)
49 const char *p
= ls
->rfunc(ls
->L
, ls
->rdata
, &sz
);
50 if (p
== NULL
|| sz
== 0) return LEX_EOF
;
53 return (LexChar
)(uint8_t)p
[0];
56 /* Get next character. */
57 static LJ_AINLINE LexChar
lex_next(LexState
*ls
)
59 return (ls
->c
= ls
->p
< ls
->pe
? (LexChar
)(uint8_t)*ls
->p
++ : lex_more(ls
));
63 static LJ_AINLINE
void lex_save(LexState
*ls
, LexChar c
)
65 lj_buf_putb(&ls
->sb
, c
);
68 /* Save previous character and get next character. */
69 static LJ_AINLINE LexChar
lex_savenext(LexState
*ls
)
75 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
76 static void lex_newline(LexState
*ls
)
79 lua_assert(lex_iseol(ls
));
80 lex_next(ls
); /* Skip "\n" or "\r". */
81 if (lex_iseol(ls
) && ls
->c
!= old
) lex_next(ls
); /* Skip "\n\r" or "\r\n". */
82 if (++ls
->linenumber
>= LJ_MAX_LINE
)
83 lj_lex_error(ls
, ls
->tok
, LJ_ERR_XLINES
);
86 /* -- Scanner for terminals ----------------------------------------------- */
88 /* Parse a number literal. */
89 static void lex_number(LexState
*ls
, TValue
*tv
)
93 lua_assert(lj_char_isdigit(ls
->c
));
94 if ((c
= ls
->c
) == '0' && (lex_savenext(ls
) | 0x20) == 'x')
96 while (lj_char_isident(ls
->c
) || ls
->c
== '.' ||
97 ((ls
->c
== '-' || ls
->c
== '+') && (c
| 0x20) == xp
)) {
102 fmt
= lj_strscan_scan((const uint8_t *)sbufB(&ls
->sb
), tv
,
103 (LJ_DUALNUM
? STRSCAN_OPT_TOINT
: STRSCAN_OPT_TONUM
) |
104 (LJ_HASFFI
? (STRSCAN_OPT_LL
|STRSCAN_OPT_IMAG
) : 0));
105 if (LJ_DUALNUM
&& fmt
== STRSCAN_INT
) {
106 setitype(tv
, LJ_TISNUM
);
107 } else if (fmt
== STRSCAN_NUM
) {
108 /* Already in correct format. */
110 } else if (fmt
!= STRSCAN_ERROR
) {
111 lua_State
*L
= ls
->L
;
113 lua_assert(fmt
== STRSCAN_I64
|| fmt
== STRSCAN_U64
|| fmt
== STRSCAN_IMAG
);
114 if (!ctype_ctsG(G(L
))) {
115 ptrdiff_t oldtop
= savestack(L
, L
->top
);
116 luaopen_ffi(L
); /* Load FFI library on-demand. */
117 L
->top
= restorestack(L
, oldtop
);
119 if (fmt
== STRSCAN_IMAG
) {
120 cd
= lj_cdata_new_(L
, CTID_COMPLEX_DOUBLE
, 2*sizeof(double));
121 ((double *)cdataptr(cd
))[0] = 0;
122 ((double *)cdataptr(cd
))[1] = numV(tv
);
124 cd
= lj_cdata_new_(L
, fmt
==STRSCAN_I64
? CTID_INT64
: CTID_UINT64
, 8);
125 *(uint64_t *)cdataptr(cd
) = tv
->u64
;
127 lj_parse_keepcdata(ls
, tv
, cd
);
130 lua_assert(fmt
== STRSCAN_ERROR
);
131 lj_lex_error(ls
, TK_number
, LJ_ERR_XNUMBER
);
135 /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
136 static int lex_skipeq(LexState
*ls
)
140 lua_assert(s
== '[' || s
== ']');
141 while (lex_savenext(ls
) == '=')
143 return (ls
->c
== s
) ? count
: (-count
) - 1;
146 /* Parse a long string or long comment (tv set to NULL). */
147 static void lex_longstring(LexState
*ls
, TValue
*tv
, int sep
)
149 lex_savenext(ls
); /* Skip second '['. */
150 if (lex_iseol(ls
)) /* Skip initial newline. */
155 lj_lex_error(ls
, TK_eof
, tv
? LJ_ERR_XLSTR
: LJ_ERR_XLCOM
);
158 if (lex_skipeq(ls
) == sep
) {
159 lex_savenext(ls
); /* Skip second ']'. */
167 if (!tv
) lj_buf_reset(&ls
->sb
); /* Don't waste space for comments. */
175 GCstr
*str
= lj_parse_keepstr(ls
, sbufB(&ls
->sb
) + (2 + (MSize
)sep
),
176 sbuflen(&ls
->sb
) - 2*(2 + (MSize
)sep
));
177 setstrV(ls
->L
, tv
, str
);
181 /* Parse a string. */
182 static void lex_string(LexState
*ls
, TValue
*tv
)
184 LexChar delim
= ls
->c
; /* Delimiter is '\'' or '"'. */
186 while (ls
->c
!= delim
) {
189 lj_lex_error(ls
, TK_eof
, LJ_ERR_XSTR
);
193 lj_lex_error(ls
, TK_string
, LJ_ERR_XSTR
);
196 LexChar c
= lex_next(ls
); /* Skip the '\\'. */
198 case 'a': c
= '\a'; break;
199 case 'b': c
= '\b'; break;
200 case 'f': c
= '\f'; break;
201 case 'n': c
= '\n'; break;
202 case 'r': c
= '\r'; break;
203 case 't': c
= '\t'; break;
204 case 'v': c
= '\v'; break;
205 case 'x': /* Hexadecimal escape '\xXX'. */
206 c
= (lex_next(ls
) & 15u) << 4;
207 if (!lj_char_isdigit(ls
->c
)) {
208 if (!lj_char_isxdigit(ls
->c
)) goto err_xesc
;
211 c
+= (lex_next(ls
) & 15u);
212 if (!lj_char_isdigit(ls
->c
)) {
213 if (!lj_char_isxdigit(ls
->c
)) goto err_xesc
;
217 case 'z': /* Skip whitespace. */
219 while (lj_char_isspace(ls
->c
))
220 if (lex_iseol(ls
)) lex_newline(ls
); else lex_next(ls
);
222 case '\n': case '\r': lex_save(ls
, '\n'); lex_newline(ls
); continue;
223 case '\\': case '\"': case '\'': break;
224 case LEX_EOF
: continue;
226 if (!lj_char_isdigit(c
))
228 c
-= '0'; /* Decimal escape '\ddd'. */
229 if (lj_char_isdigit(lex_next(ls
))) {
230 c
= c
*10 + (ls
->c
- '0');
231 if (lj_char_isdigit(lex_next(ls
))) {
232 c
= c
*10 + (ls
->c
- '0');
235 lj_lex_error(ls
, TK_string
, LJ_ERR_XESC
);
252 lex_savenext(ls
); /* Skip trailing delimiter. */
254 lj_parse_keepstr(ls
, sbufB(&ls
->sb
)+1, sbuflen(&ls
->sb
)-2));
257 /* -- Main lexical scanner ------------------------------------------------ */
259 /* Get next lexical token. */
260 static LexToken
lex_scan(LexState
*ls
, TValue
*tv
)
262 lj_buf_reset(&ls
->sb
);
264 if (lj_char_isident(ls
->c
)) {
266 if (lj_char_isdigit(ls
->c
)) { /* Numeric literal. */
270 /* Identifier or reserved word. */
273 } while (lj_char_isident(ls
->c
));
274 s
= lj_parse_keepstr(ls
, sbufB(&ls
->sb
), sbuflen(&ls
->sb
));
275 setstrV(ls
->L
, tv
, s
);
276 if (s
->reserved
> 0) /* Reserved word? */
277 return TK_OFS
+ s
->reserved
;
293 if (ls
->c
!= '-') return '-';
295 if (ls
->c
== '[') { /* Long comment "--[=*[...]=*]". */
296 int sep
= lex_skipeq(ls
);
297 lj_buf_reset(&ls
->sb
); /* `lex_skipeq' may dirty the buffer */
299 lex_longstring(ls
, NULL
, sep
);
300 lj_buf_reset(&ls
->sb
);
304 /* Short comment "--.*\n". */
305 while (!lex_iseol(ls
) && ls
->c
!= LEX_EOF
)
309 int sep
= lex_skipeq(ls
);
311 lex_longstring(ls
, tv
, sep
);
313 } else if (sep
== -1) {
316 lj_lex_error(ls
, TK_string
, LJ_ERR_XLDELIM
);
322 if (ls
->c
!= '=') return '='; else { lex_next(ls
); return TK_eq
; }
325 if (ls
->c
!= '=') return '<'; else { lex_next(ls
); return TK_le
; }
328 if (ls
->c
!= '=') return '>'; else { lex_next(ls
); return TK_ge
; }
331 if (ls
->c
!= '=') return '~'; else { lex_next(ls
); return TK_ne
; }
334 if (ls
->c
!= ':') return ':'; else { lex_next(ls
); return TK_label
; }
340 if (lex_savenext(ls
) == '.') {
344 return TK_dots
; /* ... */
346 return TK_concat
; /* .. */
347 } else if (!lj_char_isdigit(ls
->c
)) {
358 return c
; /* Single-char tokens (+ - / ...). */
364 /* -- Lexer API ----------------------------------------------------------- */
366 /* Setup lexer state. */
367 int lj_lex_setup(lua_State
*L
, LexState
*ls
)
372 ls
->pe
= ls
->p
= NULL
;
378 ls
->lookahead
= TK_eof
; /* No look-ahead token. */
381 lex_next(ls
); /* Read-ahead first char. */
382 if (ls
->c
== 0xef && ls
->p
+ 2 <= ls
->pe
&& (uint8_t)ls
->p
[0] == 0xbb &&
383 (uint8_t)ls
->p
[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
388 if (ls
->c
== '#') { /* Skip POSIX #! header line. */
391 if (ls
->c
== LEX_EOF
) return 0;
392 } while (!lex_iseol(ls
));
396 if (ls
->c
== LUA_SIGNATURE
[0]) { /* Bytecode dump. */
399 ** Loading bytecode with an extra header is disabled for security
400 ** reasons. This may circumvent the usual check for bytecode vs.
401 ** Lua code by looking at the first char. Since this is a potential
402 ** security violation no attempt is made to echo the chunkname either.
404 setstrV(L
, L
->top
++, lj_err_str(L
, LJ_ERR_BCBAD
));
405 lj_err_throw(L
, LUA_ERRSYNTAX
);
412 /* Cleanup lexer state. */
413 void lj_lex_cleanup(lua_State
*L
, LexState
*ls
)
415 global_State
*g
= G(L
);
416 lj_mem_freevec(g
, ls
->bcstack
, ls
->sizebcstack
, BCInsLine
);
417 lj_mem_freevec(g
, ls
->vstack
, ls
->sizevstack
, VarInfo
);
418 lj_buf_free(g
, &ls
->sb
);
421 /* Return next lexical token. */
422 void lj_lex_next(LexState
*ls
)
424 ls
->lastline
= ls
->linenumber
;
425 if (LJ_LIKELY(ls
->lookahead
== TK_eof
)) { /* No lookahead token? */
426 ls
->tok
= lex_scan(ls
, &ls
->tokval
); /* Get next token. */
427 } else { /* Otherwise return lookahead token. */
428 ls
->tok
= ls
->lookahead
;
429 ls
->lookahead
= TK_eof
;
430 ls
->tokval
= ls
->lookaheadval
;
434 /* Look ahead for the next token. */
435 LexToken
lj_lex_lookahead(LexState
*ls
)
437 lua_assert(ls
->lookahead
== TK_eof
);
438 ls
->lookahead
= lex_scan(ls
, &ls
->lookaheadval
);
439 return ls
->lookahead
;
442 /* Convert token to string. */
443 const char *lj_lex_token2str(LexState
*ls
, LexToken tok
)
446 return tokennames
[tok
-TK_OFS
-1];
447 else if (!lj_char_iscntrl(tok
))
448 return lj_strfmt_pushf(ls
->L
, "%c", tok
);
450 return lj_strfmt_pushf(ls
->L
, "char(%d)", tok
);
454 void lj_lex_error(LexState
*ls
, LexToken tok
, ErrMsg em
, ...)
460 } else if (tok
== TK_name
|| tok
== TK_string
|| tok
== TK_number
) {
462 tokstr
= sbufB(&ls
->sb
);
464 tokstr
= lj_lex_token2str(ls
, tok
);
467 lj_err_lex(ls
->L
, ls
->chunkname
, tokstr
, ls
->linenumber
, em
, argp
);
471 /* Initialize strings for reserved words. */
472 void lj_lex_init(lua_State
*L
)
475 for (i
= 0; i
< TK_RESERVED
; i
++) {
476 GCstr
*s
= lj_str_newz(L
, tokennames
[i
]);
477 fixstring(s
); /* Reserved words are never collected. */
478 s
->reserved
= (uint8_t)(i
+1);