fix getsup (HH)
[luatex.git] / source / libs / luajit / LuaJIT-src / src / lj_lex.c
blob0187c4267a2b5b98b56226afaa39992ec3a6cf43
1 /*
2 ** Lexical analyzer.
3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h
4 **
5 ** Major portions taken verbatim or adapted from the Lua interpreter.
6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7 */
9 #define lj_lex_c
10 #define LUA_CORE
12 #include "lj_obj.h"
13 #include "lj_gc.h"
14 #include "lj_err.h"
15 #include "lj_buf.h"
16 #include "lj_str.h"
17 #if LJ_HASFFI
18 #include "lj_tab.h"
19 #include "lj_ctype.h"
20 #include "lj_cdata.h"
21 #include "lualib.h"
22 #endif
23 #include "lj_state.h"
24 #include "lj_lex.h"
25 #include "lj_parse.h"
26 #include "lj_char.h"
27 #include "lj_strscan.h"
28 #include "lj_strfmt.h"
30 /* Lua lexer token names. */
31 static const char *const tokennames[] = {
32 #define TKSTR1(name) #name,
33 #define TKSTR2(name, sym) #sym,
34 TKDEF(TKSTR1, TKSTR2)
35 #undef TKSTR1
36 #undef TKSTR2
37 NULL
40 /* -- Buffer handling ----------------------------------------------------- */
42 #define LEX_EOF (-1)
43 #define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
45 /* Get more input from reader. */
46 static LJ_NOINLINE LexChar lex_more(LexState *ls)
48 size_t sz;
49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
50 if (p == NULL || sz == 0) return LEX_EOF;
51 ls->pe = p + sz;
52 ls->p = p + 1;
53 return (LexChar)(uint8_t)p[0];
56 /* Get next character. */
57 static LJ_AINLINE LexChar lex_next(LexState *ls)
59 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
62 /* Save character. */
63 static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
65 lj_buf_putb(&ls->sb, c);
68 /* Save previous character and get next character. */
69 static LJ_AINLINE LexChar lex_savenext(LexState *ls)
71 lex_save(ls, ls->c);
72 return lex_next(ls);
75 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
76 static void lex_newline(LexState *ls)
78 LexChar old = ls->c;
79 lua_assert(lex_iseol(ls));
80 lex_next(ls); /* Skip "\n" or "\r". */
81 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
82 if (++ls->linenumber >= LJ_MAX_LINE)
83 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
86 /* -- Scanner for terminals ----------------------------------------------- */
88 /* Parse a number literal. */
89 static void lex_number(LexState *ls, TValue *tv)
91 StrScanFmt fmt;
92 LexChar c, xp = 'e';
93 lua_assert(lj_char_isdigit(ls->c));
94 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
95 xp = 'p';
96 while (lj_char_isident(ls->c) || ls->c == '.' ||
97 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
98 c = ls->c;
99 lex_savenext(ls);
101 lex_save(ls, '\0');
102 fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
103 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
104 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
105 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
106 setitype(tv, LJ_TISNUM);
107 } else if (fmt == STRSCAN_NUM) {
108 /* Already in correct format. */
109 #if LJ_HASFFI
110 } else if (fmt != STRSCAN_ERROR) {
111 lua_State *L = ls->L;
112 GCcdata *cd;
113 lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG);
114 if (!ctype_ctsG(G(L))) {
115 ptrdiff_t oldtop = savestack(L, L->top);
116 luaopen_ffi(L); /* Load FFI library on-demand. */
117 L->top = restorestack(L, oldtop);
119 if (fmt == STRSCAN_IMAG) {
120 cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
121 ((double *)cdataptr(cd))[0] = 0;
122 ((double *)cdataptr(cd))[1] = numV(tv);
123 } else {
124 cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
125 *(uint64_t *)cdataptr(cd) = tv->u64;
127 lj_parse_keepcdata(ls, tv, cd);
128 #endif
129 } else {
130 lua_assert(fmt == STRSCAN_ERROR);
131 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
135 /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
136 static int lex_skipeq(LexState *ls)
138 int count = 0;
139 LexChar s = ls->c;
140 lua_assert(s == '[' || s == ']');
141 while (lex_savenext(ls) == '=')
142 count++;
143 return (ls->c == s) ? count : (-count) - 1;
146 /* Parse a long string or long comment (tv set to NULL). */
147 static void lex_longstring(LexState *ls, TValue *tv, int sep)
149 lex_savenext(ls); /* Skip second '['. */
150 if (lex_iseol(ls)) /* Skip initial newline. */
151 lex_newline(ls);
152 for (;;) {
153 switch (ls->c) {
154 case LEX_EOF:
155 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
156 break;
157 case ']':
158 if (lex_skipeq(ls) == sep) {
159 lex_savenext(ls); /* Skip second ']'. */
160 goto endloop;
162 break;
163 case '\n':
164 case '\r':
165 lex_save(ls, '\n');
166 lex_newline(ls);
167 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
168 break;
169 default:
170 lex_savenext(ls);
171 break;
173 } endloop:
174 if (tv) {
175 GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
176 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
177 setstrV(ls->L, tv, str);
181 /* Parse a string. */
182 static void lex_string(LexState *ls, TValue *tv)
184 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
185 lex_savenext(ls);
186 while (ls->c != delim) {
187 switch (ls->c) {
188 case LEX_EOF:
189 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
190 continue;
191 case '\n':
192 case '\r':
193 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
194 continue;
195 case '\\': {
196 LexChar c = lex_next(ls); /* Skip the '\\'. */
197 switch (c) {
198 case 'a': c = '\a'; break;
199 case 'b': c = '\b'; break;
200 case 'f': c = '\f'; break;
201 case 'n': c = '\n'; break;
202 case 'r': c = '\r'; break;
203 case 't': c = '\t'; break;
204 case 'v': c = '\v'; break;
205 case 'x': /* Hexadecimal escape '\xXX'. */
206 c = (lex_next(ls) & 15u) << 4;
207 if (!lj_char_isdigit(ls->c)) {
208 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
209 c += 9 << 4;
211 c += (lex_next(ls) & 15u);
212 if (!lj_char_isdigit(ls->c)) {
213 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
214 c += 9;
216 break;
217 case 'u': /* Unicode escape '\u{XX...}'. */
218 if (lex_next(ls) != '{') goto err_xesc;
219 lex_next(ls);
220 c = 0;
221 do {
222 c = (c << 4) | (ls->c & 15u);
223 if (!lj_char_isdigit(ls->c)) {
224 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
225 c += 9;
227 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
228 } while (lex_next(ls) != '}');
229 if (c < 0x800) {
230 if (c < 0x80) break;
231 lex_save(ls, 0xc0 | (c >> 6));
232 } else {
233 if (c >= 0x10000) {
234 lex_save(ls, 0xf0 | (c >> 18));
235 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
236 } else {
237 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
238 lex_save(ls, 0xe0 | (c >> 12));
240 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
242 c = 0x80 | (c & 0x3f);
243 break;
244 case 'z': /* Skip whitespace. */
245 lex_next(ls);
246 while (lj_char_isspace(ls->c))
247 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
248 continue;
249 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
250 case '\\': case '\"': case '\'': break;
251 case LEX_EOF: continue;
252 default:
253 if (!lj_char_isdigit(c))
254 goto err_xesc;
255 c -= '0'; /* Decimal escape '\ddd'. */
256 if (lj_char_isdigit(lex_next(ls))) {
257 c = c*10 + (ls->c - '0');
258 if (lj_char_isdigit(lex_next(ls))) {
259 c = c*10 + (ls->c - '0');
260 if (c > 255) {
261 err_xesc:
262 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
264 lex_next(ls);
267 lex_save(ls, c);
268 continue;
270 lex_save(ls, c);
271 lex_next(ls);
272 continue;
274 default:
275 lex_savenext(ls);
276 break;
279 lex_savenext(ls); /* Skip trailing delimiter. */
280 setstrV(ls->L, tv,
281 lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
284 /* -- Main lexical scanner ------------------------------------------------ */
286 /* Get next lexical token. */
287 static LexToken lex_scan(LexState *ls, TValue *tv)
289 lj_buf_reset(&ls->sb);
290 for (;;) {
291 if (lj_char_isident(ls->c)) {
292 GCstr *s;
293 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
294 lex_number(ls, tv);
295 return TK_number;
297 /* Identifier or reserved word. */
298 do {
299 lex_savenext(ls);
300 } while (lj_char_isident(ls->c));
301 s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
302 setstrV(ls->L, tv, s);
303 if (s->reserved > 0) /* Reserved word? */
304 return TK_OFS + s->reserved;
305 return TK_name;
307 switch (ls->c) {
308 case '\n':
309 case '\r':
310 lex_newline(ls);
311 continue;
312 case ' ':
313 case '\t':
314 case '\v':
315 case '\f':
316 lex_next(ls);
317 continue;
318 case '-':
319 lex_next(ls);
320 if (ls->c != '-') return '-';
321 lex_next(ls);
322 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
323 int sep = lex_skipeq(ls);
324 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
325 if (sep >= 0) {
326 lex_longstring(ls, NULL, sep);
327 lj_buf_reset(&ls->sb);
328 continue;
331 /* Short comment "--.*\n". */
332 while (!lex_iseol(ls) && ls->c != LEX_EOF)
333 lex_next(ls);
334 continue;
335 case '[': {
336 int sep = lex_skipeq(ls);
337 if (sep >= 0) {
338 lex_longstring(ls, tv, sep);
339 return TK_string;
340 } else if (sep == -1) {
341 return '[';
342 } else {
343 lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
344 continue;
347 case '=':
348 lex_next(ls);
349 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
350 case '<':
351 lex_next(ls);
352 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
353 case '>':
354 lex_next(ls);
355 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
356 case '~':
357 lex_next(ls);
358 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
359 case ':':
360 lex_next(ls);
361 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
362 case '"':
363 case '\'':
364 lex_string(ls, tv);
365 return TK_string;
366 case '.':
367 if (lex_savenext(ls) == '.') {
368 lex_next(ls);
369 if (ls->c == '.') {
370 lex_next(ls);
371 return TK_dots; /* ... */
373 return TK_concat; /* .. */
374 } else if (!lj_char_isdigit(ls->c)) {
375 return '.';
376 } else {
377 lex_number(ls, tv);
378 return TK_number;
380 case LEX_EOF:
381 return TK_eof;
382 default: {
383 LexChar c = ls->c;
384 lex_next(ls);
385 return c; /* Single-char tokens (+ - / ...). */
391 /* -- Lexer API ----------------------------------------------------------- */
393 /* Setup lexer state. */
394 int lj_lex_setup(lua_State *L, LexState *ls)
396 int header = 0;
397 ls->L = L;
398 ls->fs = NULL;
399 ls->pe = ls->p = NULL;
400 ls->vstack = NULL;
401 ls->sizevstack = 0;
402 ls->vtop = 0;
403 ls->bcstack = NULL;
404 ls->sizebcstack = 0;
405 ls->tok = 0;
406 ls->lookahead = TK_eof; /* No look-ahead token. */
407 ls->linenumber = 1;
408 ls->lastline = 1;
409 lex_next(ls); /* Read-ahead first char. */
410 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
411 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
412 ls->p += 2;
413 lex_next(ls);
414 header = 1;
416 if (ls->c == '#') { /* Skip POSIX #! header line. */
417 do {
418 lex_next(ls);
419 if (ls->c == LEX_EOF) return 0;
420 } while (!lex_iseol(ls));
421 lex_newline(ls);
422 header = 1;
424 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
425 if (header) {
427 ** Loading bytecode with an extra header is disabled for security
428 ** reasons. This may circumvent the usual check for bytecode vs.
429 ** Lua code by looking at the first char. Since this is a potential
430 ** security violation no attempt is made to echo the chunkname either.
432 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
433 lj_err_throw(L, LUA_ERRSYNTAX);
435 return 1;
437 return 0;
440 /* Cleanup lexer state. */
441 void lj_lex_cleanup(lua_State *L, LexState *ls)
443 global_State *g = G(L);
444 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
445 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
446 lj_buf_free(g, &ls->sb);
449 /* Return next lexical token. */
450 void lj_lex_next(LexState *ls)
452 ls->lastline = ls->linenumber;
453 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
454 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
455 } else { /* Otherwise return lookahead token. */
456 ls->tok = ls->lookahead;
457 ls->lookahead = TK_eof;
458 ls->tokval = ls->lookaheadval;
462 /* Look ahead for the next token. */
463 LexToken lj_lex_lookahead(LexState *ls)
465 lua_assert(ls->lookahead == TK_eof);
466 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
467 return ls->lookahead;
470 /* Convert token to string. */
471 const char *lj_lex_token2str(LexState *ls, LexToken tok)
473 if (tok > TK_OFS)
474 return tokennames[tok-TK_OFS-1];
475 else if (!lj_char_iscntrl(tok))
476 return lj_strfmt_pushf(ls->L, "%c", tok);
477 else
478 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
481 /* Lexer error. */
482 void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
484 const char *tokstr;
485 va_list argp;
486 if (tok == 0) {
487 tokstr = NULL;
488 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
489 lex_save(ls, '\0');
490 tokstr = sbufB(&ls->sb);
491 } else {
492 tokstr = lj_lex_token2str(ls, tok);
494 va_start(argp, em);
495 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
496 va_end(argp);
499 /* Initialize strings for reserved words. */
500 void lj_lex_init(lua_State *L)
502 uint32_t i;
503 for (i = 0; i < TK_RESERVED; i++) {
504 GCstr *s = lj_str_newz(L, tokennames[i]);
505 fixstring(s); /* Reserved words are never collected. */
506 s->reserved = (uint8_t)(i+1);