3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
5 ** Major portions taken verbatim or adapted from the Lua interpreter.
6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
27 /* Lua lexer token names. */
28 static const char *const tokennames
[] = {
29 #define TKSTR1(name) #name,
30 #define TKSTR2(name, sym) #sym,
37 /* -- Buffer handling ----------------------------------------------------- */
39 #define char2int(c) ((int)(uint8_t)(c))
41 (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
42 #define save_and_next(ls) (save(ls, ls->current), next(ls))
43 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
44 #define END_OF_STREAM (-1)
46 static int fillbuf(LexState
*ls
)
49 const char *buf
= ls
->rfunc(ls
->L
, ls
->rdata
, &sz
);
50 if (buf
== NULL
|| sz
== 0) return END_OF_STREAM
;
51 ls
->n
= (MSize
)sz
- 1;
53 return char2int(*(ls
->p
++));
56 static LJ_NOINLINE
void save_grow(LexState
*ls
, int c
)
59 if (ls
->sb
.sz
>= LJ_MAX_STR
/2)
60 lj_lex_error(ls
, 0, LJ_ERR_XELEM
);
61 newsize
= ls
->sb
.sz
* 2;
62 lj_str_resizebuf(ls
->L
, &ls
->sb
, newsize
);
63 ls
->sb
.buf
[ls
->sb
.n
++] = (char)c
;
66 static LJ_AINLINE
void save(LexState
*ls
, int c
)
68 if (LJ_UNLIKELY(ls
->sb
.n
+ 1 > ls
->sb
.sz
))
71 ls
->sb
.buf
[ls
->sb
.n
++] = (char)c
;
74 static void inclinenumber(LexState
*ls
)
76 int old
= ls
->current
;
77 lua_assert(currIsNewline(ls
));
78 next(ls
); /* skip `\n' or `\r' */
79 if (currIsNewline(ls
) && ls
->current
!= old
)
80 next(ls
); /* skip `\n\r' or `\r\n' */
81 if (++ls
->linenumber
>= LJ_MAX_LINE
)
82 lj_lex_error(ls
, ls
->token
, LJ_ERR_XLINES
);
85 /* -- Scanner for terminals ----------------------------------------------- */
88 /* Load FFI library on-demand. Needed if we create cdata objects. */
89 static void lex_loadffi(lua_State
*L
)
91 ptrdiff_t oldtop
= savestack(L
, L
->top
);
93 L
->top
= restorestack(L
, oldtop
);
96 /* Parse 64 bit integer. */
97 static int lex_number64(LexState
*ls
, TValue
*tv
)
100 uint8_t *p
= (uint8_t *)ls
->sb
.buf
;
101 CTypeID id
= CTID_INT64
;
104 if (p
[0] == '0' && (p
[1] & ~0x20) == 'X') { /* Hexadecimal. */
106 if (!lj_char_isxdigit(*p
)) return 0;
108 n
= n
*16 + (*p
& 15);
109 if (!lj_char_isdigit(*p
)) n
+= 9;
111 } while (lj_char_isxdigit(*p
));
112 } else { /* Decimal. */
113 if (!lj_char_isdigit(*p
)) return 0;
115 n
= n
*10 + (*p
- '0');
117 } while (lj_char_isdigit(*p
));
119 for (;;) { /* Parse suffixes. */
120 if ((*p
& ~0x20) == 'U')
122 else if ((*p
& ~0x20) == 'L')
128 if (numl
!= 2 || *p
!= '\0') return 0;
129 /* Return cdata holding a 64 bit integer. */
130 cd
= lj_cdata_new_(ls
->L
, id
, 8);
131 *(uint64_t *)cdataptr(cd
) = n
;
132 lj_parse_keepcdata(ls
, tv
, cd
);
137 /* Parse a number literal. */
138 static void lex_number(LexState
*ls
, TValue
*tv
)
141 lua_assert(lj_char_isdigit(ls
->current
));
145 } while (lj_char_isident(ls
->current
) || ls
->current
== '.' ||
146 ((ls
->current
== '-' || ls
->current
== '+') &&
147 ((c
& ~0x20) == 'E' || (c
& ~0x20) == 'P')));
150 if ((c
== 'I' || c
== 'L' || c
== 'U') && !ctype_ctsG(G(ls
->L
)))
152 if (c
== 'I') /* Parse imaginary part of complex number. */
157 if ((c
== 'L' || c
== 'U') && lex_number64(ls
, tv
)) { /* Parse 64 bit int. */
161 if (lj_str_numconv(ls
->sb
.buf
, tv
)) {
163 if (c
== 'I') { /* Return cdata holding a complex number. */
164 GCcdata
*cd
= lj_cdata_new_(ls
->L
, CTID_COMPLEX_DOUBLE
, 2*sizeof(double));
165 ((double *)cdataptr(cd
))[0] = 0;
166 ((double *)cdataptr(cd
))[1] = numberVnum(tv
);
167 lj_parse_keepcdata(ls
, tv
, cd
);
170 if (LJ_DUALNUM
&& tvisnum(tv
)) {
171 int32_t k
= lj_num2int(numV(tv
));
172 if ((lua_Number
)k
== numV(tv
)) /* -0 cannot end up here. */
177 lj_lex_error(ls
, TK_number
, LJ_ERR_XNUMBER
);
180 static int skip_sep(LexState
*ls
)
184 lua_assert(s
== '[' || s
== ']');
186 while (ls
->current
== '=') {
190 return (ls
->current
== s
) ? count
: (-count
) - 1;
193 static void read_long_string(LexState
*ls
, TValue
*tv
, int sep
)
195 save_and_next(ls
); /* skip 2nd `[' */
196 if (currIsNewline(ls
)) /* string starts with a newline? */
197 inclinenumber(ls
); /* skip it */
199 switch (ls
->current
) {
201 lj_lex_error(ls
, TK_eof
, tv
? LJ_ERR_XLSTR
: LJ_ERR_XLCOM
);
204 if (skip_sep(ls
) == sep
) {
205 save_and_next(ls
); /* skip 2nd `]' */
213 if (!tv
) lj_str_resetbuf(&ls
->sb
); /* avoid wasting space */
216 if (tv
) save_and_next(ls
);
222 GCstr
*str
= lj_parse_keepstr(ls
, ls
->sb
.buf
+ (2 + (MSize
)sep
),
223 ls
->sb
.n
- 2*(2 + (MSize
)sep
));
224 setstrV(ls
->L
, tv
, str
);
228 static void read_string(LexState
*ls
, int delim
, TValue
*tv
)
231 while (ls
->current
!= delim
) {
232 switch (ls
->current
) {
234 lj_lex_error(ls
, TK_eof
, LJ_ERR_XSTR
);
238 lj_lex_error(ls
, TK_string
, LJ_ERR_XSTR
);
241 int c
= next(ls
); /* Skip the '\\'. */
243 case 'a': c
= '\a'; break;
244 case 'b': c
= '\b'; break;
245 case 'f': c
= '\f'; break;
246 case 'n': c
= '\n'; break;
247 case 'r': c
= '\r'; break;
248 case 't': c
= '\t'; break;
249 case 'v': c
= '\v'; break;
250 case 'x': /* Hexadecimal escape '\xXX'. */
251 c
= (next(ls
) & 15u) << 4;
252 if (!lj_char_isdigit(ls
->current
)) {
253 if (!lj_char_isxdigit(ls
->current
)) goto err_xesc
;
256 c
+= (next(ls
) & 15u);
257 if (!lj_char_isdigit(ls
->current
)) {
258 if (!lj_char_isxdigit(ls
->current
)) goto err_xesc
;
262 case 'z': /* Skip whitespace. */
264 while (lj_char_isspace(ls
->current
))
265 if (currIsNewline(ls
)) inclinenumber(ls
); else next(ls
);
267 case '\n': case '\r': save(ls
, '\n'); inclinenumber(ls
); continue;
268 case '\\': case '\"': case '\'': break;
269 case END_OF_STREAM
: continue;
271 if (!lj_char_isdigit(c
))
273 c
-= '0'; /* Decimal escape '\ddd'. */
274 if (lj_char_isdigit(next(ls
))) {
275 c
= c
*10 + (ls
->current
- '0');
276 if (lj_char_isdigit(next(ls
))) {
277 c
= c
*10 + (ls
->current
- '0');
280 lj_lex_error(ls
, TK_string
, LJ_ERR_XESC
);
297 save_and_next(ls
); /* skip delimiter */
298 setstrV(ls
->L
, tv
, lj_parse_keepstr(ls
, ls
->sb
.buf
+ 1, ls
->sb
.n
- 2));
301 /* -- Main lexical scanner ------------------------------------------------ */
303 static int llex(LexState
*ls
, TValue
*tv
)
305 lj_str_resetbuf(&ls
->sb
);
307 if (lj_char_isident(ls
->current
)) {
309 if (lj_char_isdigit(ls
->current
)) { /* Numeric literal. */
313 /* Identifier or reserved word. */
316 } while (lj_char_isident(ls
->current
));
317 s
= lj_parse_keepstr(ls
, ls
->sb
.buf
, ls
->sb
.n
);
318 if (s
->reserved
> 0) /* Reserved word? */
319 return TK_OFS
+ s
->reserved
;
320 setstrV(ls
->L
, tv
, s
);
323 switch (ls
->current
) {
336 if (ls
->current
!= '-') return '-';
337 /* else is a comment */
339 if (ls
->current
== '[') {
340 int sep
= skip_sep(ls
);
341 lj_str_resetbuf(&ls
->sb
); /* `skip_sep' may dirty the buffer */
343 read_long_string(ls
, NULL
, sep
); /* long comment */
344 lj_str_resetbuf(&ls
->sb
);
348 /* else short comment */
349 while (!currIsNewline(ls
) && ls
->current
!= END_OF_STREAM
)
353 int sep
= skip_sep(ls
);
355 read_long_string(ls
, tv
, sep
);
357 } else if (sep
== -1) {
360 lj_lex_error(ls
, TK_string
, LJ_ERR_XLDELIM
);
366 if (ls
->current
!= '=') return '='; else { next(ls
); return TK_eq
; }
369 if (ls
->current
!= '=') return '<'; else { next(ls
); return TK_le
; }
372 if (ls
->current
!= '=') return '>'; else { next(ls
); return TK_ge
; }
375 if (ls
->current
!= '=') return '~'; else { next(ls
); return TK_ne
; }
378 read_string(ls
, ls
->current
, tv
);
382 if (ls
->current
== '.') {
384 if (ls
->current
== '.') {
386 return TK_dots
; /* ... */
388 return TK_concat
; /* .. */
389 } else if (!lj_char_isdigit(ls
->current
)) {
400 return c
; /* Single-char tokens (+ - / ...). */
406 /* -- Lexer API ----------------------------------------------------------- */
408 /* Setup lexer state. */
409 int lj_lex_setup(lua_State
*L
, LexState
*ls
)
420 ls
->lookahead
= TK_eof
; /* No look-ahead token. */
423 lj_str_resizebuf(ls
->L
, &ls
->sb
, LJ_MIN_SBUF
);
424 next(ls
); /* Read-ahead first char. */
425 if (ls
->current
== 0xef && ls
->n
>= 2 && char2int(ls
->p
[0]) == 0xbb &&
426 char2int(ls
->p
[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
431 if (ls
->current
== '#') { /* Skip POSIX #! header line. */
434 if (ls
->current
== END_OF_STREAM
) return 0;
435 } while (!currIsNewline(ls
));
438 return (ls
->current
== LUA_SIGNATURE
[0]); /* Bytecode dump? */
441 /* Cleanup lexer state. */
442 void lj_lex_cleanup(lua_State
*L
, LexState
*ls
)
444 global_State
*g
= G(L
);
445 lj_mem_freevec(g
, ls
->bcstack
, ls
->sizebcstack
, BCInsLine
);
446 lj_mem_freevec(g
, ls
->vstack
, ls
->sizevstack
, VarInfo
);
447 lj_str_freebuf(g
, &ls
->sb
);
450 void lj_lex_next(LexState
*ls
)
452 ls
->lastline
= ls
->linenumber
;
453 if (LJ_LIKELY(ls
->lookahead
== TK_eof
)) { /* No lookahead token? */
454 ls
->token
= llex(ls
, &ls
->tokenval
); /* Get next token. */
455 } else { /* Otherwise return lookahead token. */
456 ls
->token
= ls
->lookahead
;
457 ls
->lookahead
= TK_eof
;
458 ls
->tokenval
= ls
->lookaheadval
;
462 LexToken
lj_lex_lookahead(LexState
*ls
)
464 lua_assert(ls
->lookahead
== TK_eof
);
465 ls
->lookahead
= llex(ls
, &ls
->lookaheadval
);
466 return ls
->lookahead
;
469 const char *lj_lex_token2str(LexState
*ls
, LexToken token
)
472 return tokennames
[token
-TK_OFS
-1];
473 else if (!lj_char_iscntrl(token
))
474 return lj_str_pushf(ls
->L
, "%c", token
);
476 return lj_str_pushf(ls
->L
, "char(%d)", token
);
479 void lj_lex_error(LexState
*ls
, LexToken token
, ErrMsg em
, ...)
485 } else if (token
== TK_name
|| token
== TK_string
|| token
== TK_number
) {
489 tok
= lj_lex_token2str(ls
, token
);
492 lj_err_lex(ls
->L
, ls
->chunkname
, tok
, ls
->linenumber
, em
, argp
);
496 void lj_lex_init(lua_State
*L
)
499 for (i
= 0; i
< TK_RESERVED
; i
++) {
500 GCstr
*s
= lj_str_newz(L
, tokennames
[i
]);
501 fixstring(s
); /* Reserved words are never collected. */
502 s
->reserved
= (uint8_t)(i
+1);