?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
beta-0.89.2
[luatex.git] / source / libs / luajit / LuaJIT-src / src / lj_trace.c
blob42f4321d5d2f7520f90c5609945c1ef977531578
1 /*
2 ** Trace management.
3 ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_trace_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10
11 #if LJ_HASJIT
12
13 #include "lj_gc.h"
14 #include "lj_err.h"
15 #include "lj_debug.h"
16 #include "lj_str.h"
17 #include "lj_frame.h"
18 #include "lj_state.h"
19 #include "lj_bc.h"
20 #include "lj_ir.h"
21 #include "lj_jit.h"
22 #include "lj_iropt.h"
23 #include "lj_mcode.h"
24 #include "lj_trace.h"
25 #include "lj_snap.h"
26 #include "lj_gdbjit.h"
27 #include "lj_record.h"
28 #include "lj_asm.h"
29 #include "lj_dispatch.h"
30 #include "lj_vm.h"
31 #include "lj_vmevent.h"
32 #include "lj_target.h"
33
34 /* -- Error handling ------------------------------------------------------ */
35
36 /* Synchronous abort with error message. */
37 void lj_trace_err(jit_State *J, TraceError e)
38 {
39 setnilV(&J->errinfo); /* No error info. */
40 setintV(J->L->top++, (int32_t)e);
41 lj_err_throw(J->L, LUA_ERRRUN);
42 }
43
44 /* Synchronous abort with error message and error info. */
45 void lj_trace_err_info(jit_State *J, TraceError e)
46 {
47 setintV(J->L->top++, (int32_t)e);
48 lj_err_throw(J->L, LUA_ERRRUN);
49 }
50
51 /* -- Trace management ---------------------------------------------------- */
52
53 /* The current trace is first assembled in J->cur. The variable length
54 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
55 ** recording ends successfully, the current trace and its data structures
56 ** are copied to a new (compact) GCtrace object.
57 */
58
59 /* Find a free trace number. */
60 static TraceNo trace_findfree(jit_State *J)
61 {
62 MSize osz, lim;
63 if (J->freetrace == 0)
64 J->freetrace = 1;
65 for (; J->freetrace < J->sizetrace; J->freetrace++)
66 if (traceref(J, J->freetrace) == NULL)
67 return J->freetrace++;
68 /* Need to grow trace array. */
69 lim = (MSize)J->param[JIT_P_maxtrace] + 1;
70 if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
71 osz = J->sizetrace;
72 if (osz >= lim)
73 return 0; /* Too many traces. */
74 lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
75 for (; osz < J->sizetrace; osz++)
76 setgcrefnull(J->trace[osz]);
77 return J->freetrace;
78 }
79
80 #define TRACE_APPENDVEC(field, szfield, tp) \
81 T->field = (tp *)p; \
82 memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
83 p += J->cur.szfield*sizeof(tp);
84
85 #ifdef LUAJIT_USE_PERFTOOLS
86 /*
87 ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
88 ** Example usage:
89 ** perf record -f -e cycles luajit test.lua
90 ** perf report -s symbol
91 ** rm perf.data /tmp/perf-*.map
92 */
93 #include <stdio.h>
94 #include <unistd.h>
95
96 static void perftools_addtrace(GCtrace *T)
97 {
98 static FILE *fp;
99 GCproto *pt = &gcref(T->startpt)->pt;
100 const BCIns *startpc = mref(T->startpc, const BCIns);
101 const char *name = proto_chunknamestr(pt);
102 BCLine lineno;
103 if (name[0] == '@' || name[0] == '=')
104 name++;
105 else
106 name = "(string)";
107 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
108 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
109 if (!fp) {
110 char fname[40];
111 sprintf(fname, "/tmp/perf-%d.map", getpid());
112 if (!(fp = fopen(fname, "w"))) return;
113 setlinebuf(fp);
114 }
115 fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
116 (long)T->mcode, T->szmcode, T->traceno, name, lineno);
117 }
118 #endif
119
120 /* Allocate space for copy of trace. */
121 static GCtrace *trace_save_alloc(jit_State *J)
122 {
123 size_t sztr = ((sizeof(GCtrace)+7)&~7);
124 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
125 size_t sz = sztr + szins +
126 J->cur.nsnap*sizeof(SnapShot) +
127 J->cur.nsnapmap*sizeof(SnapEntry);
128 return lj_mem_newt(J->L, (MSize)sz, GCtrace);
129 }
130
131 /* Save current trace by copying and compacting it. */
132 static void trace_save(jit_State *J, GCtrace *T)
133 {
134 size_t sztr = ((sizeof(GCtrace)+7)&~7);
135 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
136 char *p = (char *)T + sztr;
137 memcpy(T, &J->cur, sizeof(GCtrace));
138 setgcrefr(T->nextgc, J2G(J)->gc.root);
139 setgcrefp(J2G(J)->gc.root, T);
140 newwhite(J2G(J), T);
141 T->gct = ~LJ_TTRACE;
142 T->ir = (IRIns *)p - J->cur.nk;
143 memcpy(p, J->cur.ir+J->cur.nk, szins);
144 p += szins;
145 TRACE_APPENDVEC(snap, nsnap, SnapShot)
146 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
147 J->cur.traceno = 0;
148 setgcrefp(J->trace[T->traceno], T);
149 lj_gc_barriertrace(J2G(J), T->traceno);
150 lj_gdbjit_addtrace(J, T);
151 #ifdef LUAJIT_USE_PERFTOOLS
152 perftools_addtrace(T);
153 #endif
154 }
155
156 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
157 {
158 jit_State *J = G2J(g);
159 if (T->traceno) {
160 lj_gdbjit_deltrace(J, T);
161 if (T->traceno < J->freetrace)
162 J->freetrace = T->traceno;
163 setgcrefnull(J->trace[T->traceno]);
164 }
165 lj_mem_free(g, T,
166 ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
167 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
168 }
169
170 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
171 void lj_trace_reenableproto(GCproto *pt)
172 {
173 if ((pt->flags & PROTO_ILOOP)) {
174 BCIns *bc = proto_bc(pt);
175 BCPos i, sizebc = pt->sizebc;;
176 pt->flags &= ~PROTO_ILOOP;
177 if (bc_op(bc[0]) == BC_IFUNCF)
178 setbc_op(&bc[0], BC_FUNCF);
179 for (i = 1; i < sizebc; i++) {
180 BCOp op = bc_op(bc[i]);
181 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
182 setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
183 }
184 }
185 }
186
187 /* Unpatch the bytecode modified by a root trace. */
188 static void trace_unpatch(jit_State *J, GCtrace *T)
189 {
190 BCOp op = bc_op(T->startins);
191 BCIns *pc = mref(T->startpc, BCIns);
192 UNUSED(J);
193 if (op == BC_JMP)
194 return; /* No need to unpatch branches in parent traces (yet). */
195 switch (bc_op(*pc)) {
196 case BC_JFORL:
197 lua_assert(traceref(J, bc_d(*pc)) == T);
198 *pc = T->startins;
199 pc += bc_j(T->startins);
200 lua_assert(bc_op(*pc) == BC_JFORI);
201 setbc_op(pc, BC_FORI);
202 break;
203 case BC_JITERL:
204 case BC_JLOOP:
205 lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
206 *pc = T->startins;
207 break;
208 case BC_JMP:
209 lua_assert(op == BC_ITERL);
210 pc += bc_j(*pc)+2;
211 if (bc_op(*pc) == BC_JITERL) {
212 lua_assert(traceref(J, bc_d(*pc)) == T);
213 *pc = T->startins;
214 }
215 break;
216 case BC_JFUNCF:
217 lua_assert(op == BC_FUNCF);
218 *pc = T->startins;
219 break;
220 default: /* Already unpatched. */
221 break;
222 }
223 }
224
225 /* Flush a root trace. */
226 static void trace_flushroot(jit_State *J, GCtrace *T)
227 {
228 GCproto *pt = &gcref(T->startpt)->pt;
229 lua_assert(T->root == 0 && pt != NULL);
230 /* First unpatch any modified bytecode. */
231 trace_unpatch(J, T);
232 /* Unlink root trace from chain anchored in prototype. */
233 if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */
234 pt->trace = T->nextroot;
235 } else if (pt->trace) { /* Otherwise search in chain of root traces. */
236 GCtrace *T2 = traceref(J, pt->trace);
237 if (T2) {
238 for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
239 if (T2->nextroot == T->traceno) {
240 T2->nextroot = T->nextroot; /* Unlink from chain. */
241 break;
242 }
243 }
244 }
245 }
246
247 /* Flush a trace. Only root traces are considered. */
248 void lj_trace_flush(jit_State *J, TraceNo traceno)
249 {
250 if (traceno > 0 && traceno < J->sizetrace) {
251 GCtrace *T = traceref(J, traceno);
252 if (T && T->root == 0)
253 trace_flushroot(J, T);
254 }
255 }
256
257 /* Flush all traces associated with a prototype. */
258 void lj_trace_flushproto(global_State *g, GCproto *pt)
259 {
260 while (pt->trace != 0)
261 trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
262 }
263
264 /* Flush all traces. */
265 int lj_trace_flushall(lua_State *L)
266 {
267 jit_State *J = L2J(L);
268 ptrdiff_t i;
269 if ((J2G(J)->hookmask & HOOK_GC))
270 return 1;
271 for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
272 GCtrace *T = traceref(J, i);
273 if (T) {
274 if (T->root == 0)
275 trace_flushroot(J, T);
276 lj_gdbjit_deltrace(J, T);
277 T->traceno = 0;
278 setgcrefnull(J->trace[i]);
279 }
280 }
281 J->cur.traceno = 0;
282 J->freetrace = 0;
283 /* Clear penalty cache. */
284 memset(J->penalty, 0, sizeof(J->penalty));
285 /* Free the whole machine code and invalidate all exit stub groups. */
286 lj_mcode_free(J);
287 memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
288 lj_vmevent_send(L, TRACE,
289 setstrV(L, L->top++, lj_str_newlit(L, "flush"));
290 );
291 return 0;
292 }
293
294 /* Initialize JIT compiler state. */
295 void lj_trace_initstate(global_State *g)
296 {
297 jit_State *J = G2J(g);
298 TValue *tv;
299 /* Initialize SIMD constants. */
300 tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
301 tv[0].u64 = U64x(7fffffff,ffffffff);
302 tv[1].u64 = U64x(7fffffff,ffffffff);
303 tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
304 tv[0].u64 = U64x(80000000,00000000);
305 tv[1].u64 = U64x(80000000,00000000);
306 }
307
308 /* Free everything associated with the JIT compiler state. */
309 void lj_trace_freestate(global_State *g)
310 {
311 jit_State *J = G2J(g);
312 #ifdef LUA_USE_ASSERT
313 { /* This assumes all traces have already been freed. */
314 ptrdiff_t i;
315 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
316 lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
317 }
318 #endif
319 lj_mcode_free(J);
320 lj_ir_k64_freeall(J);
321 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
322 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
323 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
324 lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
325 }
326
327 /* -- Penalties and blacklisting ------------------------------------------ */
328
329 /* Blacklist a bytecode instruction. */
330 static void blacklist_pc(GCproto *pt, BCIns *pc)
331 {
332 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
333 pt->flags |= PROTO_ILOOP;
334 }
335
336 /* Penalize a bytecode instruction. */
337 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
338 {
339 uint32_t i, val = PENALTY_MIN;
340 for (i = 0; i < PENALTY_SLOTS; i++)
341 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */
342 /* First try to bump its hotcount several times. */
343 val = ((uint32_t)J->penalty[i].val << 1) +
344 LJ_PRNG_BITS(J, PENALTY_RNDBITS);
345 if (val > PENALTY_MAX) {
346 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */
347 return;
348 }
349 goto setpenalty;
350 }
351 /* Assign a new penalty cache slot. */
352 i = J->penaltyslot;
353 J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
354 setmref(J->penalty[i].pc, pc);
355 setpenalty:
356 J->penalty[i].val = (uint16_t)val;
357 J->penalty[i].reason = e;
358 hotcount_set(J2GG(J), pc+1, val);
359 }
360
361 /* -- Trace compiler state machine ---------------------------------------- */
362
363 /* Start tracing. */
364 static void trace_start(jit_State *J)
365 {
366 lua_State *L;
367 TraceNo traceno;
368
369 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */
370 if (J->parent == 0 && J->exitno == 0) {
371 /* Lazy bytecode patching to disable hotcount events. */
372 lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
373 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
374 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
375 J->pt->flags |= PROTO_ILOOP;
376 }
377 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
378 return;
379 }
380
381 /* Get a new trace number. */
382 traceno = trace_findfree(J);
383 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */
384 lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
385 lj_trace_flushall(J->L);
386 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
387 return;
388 }
389 setgcrefp(J->trace[traceno], &J->cur);
390
391 /* Setup enough of the current trace to be able to send the vmevent. */
392 memset(&J->cur, 0, sizeof(GCtrace));
393 J->cur.traceno = traceno;
394 J->cur.nins = J->cur.nk = REF_BASE;
395 J->cur.ir = J->irbuf;
396 J->cur.snap = J->snapbuf;
397 J->cur.snapmap = J->snapmapbuf;
398 J->mergesnap = 0;
399 J->needsnap = 0;
400 J->bcskip = 0;
401 J->guardemit.irt = 0;
402 J->postproc = LJ_POST_NONE;
403 lj_resetsplit(J);
404 J->retryrec = 0;
405 setgcref(J->cur.startpt, obj2gco(J->pt));
406
407 L = J->L;
408 lj_vmevent_send(L, TRACE,
409 setstrV(L, L->top++, lj_str_newlit(L, "start"));
410 setintV(L->top++, traceno);
411 setfuncV(L, L->top++, J->fn);
412 setintV(L->top++, proto_bcpos(J->pt, J->pc));
413 if (J->parent) {
414 setintV(L->top++, J->parent);
415 setintV(L->top++, J->exitno);
416 }
417 );
418 lj_record_setup(J);
419 }
420
421 /* Stop tracing. */
422 static void trace_stop(jit_State *J)
423 {
424 BCIns *pc = mref(J->cur.startpc, BCIns);
425 BCOp op = bc_op(J->cur.startins);
426 GCproto *pt = &gcref(J->cur.startpt)->pt;
427 TraceNo traceno = J->cur.traceno;
428 GCtrace *T = trace_save_alloc(J); /* Do this first. May throw OOM. */
429 lua_State *L;
430
431 switch (op) {
432 case BC_FORL:
433 setbc_op(pc+bc_j(J->cur.startins), BC_JFORI); /* Patch FORI, too. */
434 /* fallthrough */
435 case BC_LOOP:
436 case BC_ITERL:
437 case BC_FUNCF:
438 /* Patch bytecode of starting instruction in root trace. */
439 setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
440 setbc_d(pc, traceno);
441 addroot:
442 /* Add to root trace chain in prototype. */
443 J->cur.nextroot = pt->trace;
444 pt->trace = (TraceNo1)traceno;
445 break;
446 case BC_RET:
447 case BC_RET0:
448 case BC_RET1:
449 *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
450 goto addroot;
451 case BC_JMP:
452 /* Patch exit branch in parent to side trace entry. */
453 lua_assert(J->parent != 0 && J->cur.root != 0);
454 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
455 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
456 traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
457 /* Add to side trace chain in root trace. */
458 {
459 GCtrace *root = traceref(J, J->cur.root);
460 root->nchild++;
461 J->cur.nextside = root->nextside;
462 root->nextside = (TraceNo1)traceno;
463 }
464 break;
465 case BC_CALLM:
466 case BC_CALL:
467 case BC_ITERC:
468 /* Trace stitching: patch link of previous trace. */
469 traceref(J, J->exitno)->link = traceno;
470 break;
471 default:
472 lua_assert(0);
473 break;
474 }
475
476 /* Commit new mcode only after all patching is done. */
477 lj_mcode_commit(J, J->cur.mcode);
478 J->postproc = LJ_POST_NONE;
479 trace_save(J, T);
480
481 L = J->L;
482 lj_vmevent_send(L, TRACE,
483 setstrV(L, L->top++, lj_str_newlit(L, "stop"));
484 setintV(L->top++, traceno);
485 setfuncV(L, L->top++, J->fn);
486 );
487 }
488
489 /* Start a new root trace for down-recursion. */
490 static int trace_downrec(jit_State *J)
491 {
492 /* Restart recording at the return instruction. */
493 lua_assert(J->pt != NULL);
494 lua_assert(bc_isret(bc_op(*J->pc)));
495 if (bc_op(*J->pc) == BC_RETM)
496 return 0; /* NYI: down-recursion with RETM. */
497 J->parent = 0;
498 J->exitno = 0;
499 J->state = LJ_TRACE_RECORD;
500 trace_start(J);
501 return 1;
502 }
503
504 /* Abort tracing. */
505 static int trace_abort(jit_State *J)
506 {
507 lua_State *L = J->L;
508 TraceError e = LJ_TRERR_RECERR;
509 TraceNo traceno;
510
511 J->postproc = LJ_POST_NONE;
512 lj_mcode_abort(J);
513 if (tvisnumber(L->top-1))
514 e = (TraceError)numberVint(L->top-1);
515 if (e == LJ_TRERR_MCODELM) {
516 L->top--; /* Remove error object */
517 J->state = LJ_TRACE_ASM;
518 return 1; /* Retry ASM with new MCode area. */
519 }
520 /* Penalize or blacklist starting bytecode instruction. */
521 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
522 if (J->exitno == 0) {
523 BCIns *startpc = mref(J->cur.startpc, BCIns);
524 if (e == LJ_TRERR_RETRY)
525 hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */
526 else
527 penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
528 } else {
529 traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */
530 }
531 }
532
533 /* Is there anything to abort? */
534 traceno = J->cur.traceno;
535 if (traceno) {
536 ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */
537 J->cur.link = 0;
538 J->cur.linktype = LJ_TRLINK_NONE;
539 lj_vmevent_send(L, TRACE,
540 TValue *frame;
541 const BCIns *pc;
542 GCfunc *fn;
543 setstrV(L, L->top++, lj_str_newlit(L, "abort"));
544 setintV(L->top++, traceno);
545 /* Find original Lua function call to generate a better error message. */
546 frame = J->L->base-1;
547 pc = J->pc;
548 while (!isluafunc(frame_func(frame))) {
549 pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
550 frame = frame_prev(frame);
551 }
552 fn = frame_func(frame);
553 setfuncV(L, L->top++, fn);
554 setintV(L->top++, proto_bcpos(funcproto(fn), pc));
555 copyTV(L, L->top++, restorestack(L, errobj));
556 copyTV(L, L->top++, &J->errinfo);
557 );
558 /* Drop aborted trace after the vmevent (which may still access it). */
559 setgcrefnull(J->trace[traceno]);
560 if (traceno < J->freetrace)
561 J->freetrace = traceno;
562 J->cur.traceno = 0;
563 }
564 L->top--; /* Remove error object */
565 if (e == LJ_TRERR_DOWNREC)
566 return trace_downrec(J);
567 else if (e == LJ_TRERR_MCODEAL)
568 lj_trace_flushall(L);
569 return 0;
570 }
571
572 /* Perform pending re-patch of a bytecode instruction. */
573 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
574 {
575 if (LJ_UNLIKELY(J->patchpc)) {
576 if (force || J->bcskip == 0) {
577 *J->patchpc = J->patchins;
578 J->patchpc = NULL;
579 } else {
580 J->bcskip = 0;
581 }
582 }
583 }
584
585 /* State machine for the trace compiler. Protected callback. */
586 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
587 {
588 jit_State *J = (jit_State *)ud;
589 UNUSED(dummy);
590 do {
591 retry:
592 switch (J->state) {
593 case LJ_TRACE_START:
594 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */
595 trace_start(J);
596 lj_dispatch_update(J2G(J));
597 break;
598
599 case LJ_TRACE_RECORD:
600 trace_pendpatch(J, 0);
601 setvmstate(J2G(J), RECORD);
602 lj_vmevent_send_(L, RECORD,
603 /* Save/restore tmptv state for trace recorder. */
604 TValue savetv = J2G(J)->tmptv;
605 TValue savetv2 = J2G(J)->tmptv2;
606 setintV(L->top++, J->cur.traceno);
607 setfuncV(L, L->top++, J->fn);
608 setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
609 setintV(L->top++, J->framedepth);
610 ,
611 J2G(J)->tmptv = savetv;
612 J2G(J)->tmptv2 = savetv2;
613 );
614 lj_record_ins(J);
615 break;
616
617 case LJ_TRACE_END:
618 trace_pendpatch(J, 1);
619 J->loopref = 0;
620 if ((J->flags & JIT_F_OPT_LOOP) &&
621 J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
622 setvmstate(J2G(J), OPT);
623 lj_opt_dce(J);
624 if (lj_opt_loop(J)) { /* Loop optimization failed? */
625 J->cur.link = 0;
626 J->cur.linktype = LJ_TRLINK_NONE;
627 J->loopref = J->cur.nins;
628 J->state = LJ_TRACE_RECORD; /* Try to continue recording. */
629 break;
630 }
631 J->loopref = J->chain[IR_LOOP]; /* Needed by assembler. */
632 }
633 lj_opt_split(J);
634 lj_opt_sink(J);
635 if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
636 J->state = LJ_TRACE_ASM;
637 break;
638
639 case LJ_TRACE_ASM:
640 setvmstate(J2G(J), ASM);
641 lj_asm_trace(J, &J->cur);
642 trace_stop(J);
643 setvmstate(J2G(J), INTERP);
644 J->state = LJ_TRACE_IDLE;
645 lj_dispatch_update(J2G(J));
646 return NULL;
647
648 default: /* Trace aborted asynchronously. */
649 setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
650 /* fallthrough */
651 case LJ_TRACE_ERR:
652 trace_pendpatch(J, 1);
653 if (trace_abort(J))
654 goto retry;
655 setvmstate(J2G(J), INTERP);
656 J->state = LJ_TRACE_IDLE;
657 lj_dispatch_update(J2G(J));
658 return NULL;
659 }
660 } while (J->state > LJ_TRACE_RECORD);
661 return NULL;
662 }
663
664 /* -- Event handling ------------------------------------------------------ */
665
666 /* A bytecode instruction is about to be executed. Record it. */
667 void lj_trace_ins(jit_State *J, const BCIns *pc)
668 {
669 /* Note: J->L must already be set. pc is the true bytecode PC here. */
670 J->pc = pc;
671 J->fn = curr_func(J->L);
672 J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
673 while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
674 J->state = LJ_TRACE_ERR;
675 }
676
677 /* A hotcount triggered. Start recording a root trace. */
678 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
679 {
680 /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
681 ERRNO_SAVE
682 /* Reset hotcount. */
683 hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
684 /* Only start a new trace if not recording or inside __gc call or vmevent. */
685 if (J->state == LJ_TRACE_IDLE &&
686 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
687 J->parent = 0; /* Root trace. */
688 J->exitno = 0;
689 J->state = LJ_TRACE_START;
690 lj_trace_ins(J, pc-1);
691 }
692 ERRNO_RESTORE
693 }
694
695 /* Check for a hot side exit. If yes, start recording a side trace. */
696 static void trace_hotside(jit_State *J, const BCIns *pc)
697 {
698 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
699 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
700 isluafunc(curr_func(J->L)) &&
701 snap->count != SNAPCOUNT_DONE &&
702 ++snap->count >= J->param[JIT_P_hotexit]) {
703 lua_assert(J->state == LJ_TRACE_IDLE);
704 /* J->parent is non-zero for a side trace. */
705 J->state = LJ_TRACE_START;
706 lj_trace_ins(J, pc);
707 }
708 }
709
710 /* Stitch a new trace to the previous trace. */
711 void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
712 {
713 /* Only start a new trace if not recording or inside __gc call or vmevent. */
714 if (J->state == LJ_TRACE_IDLE &&
715 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
716 J->parent = 0; /* Have to treat it like a root trace. */
717 /* J->exitno is set to the invoking trace. */
718 J->state = LJ_TRACE_START;
719 lj_trace_ins(J, pc);
720 }
721 }
722
723
724 /* Tiny struct to pass data to protected call. */
725 typedef struct ExitDataCP {
726 jit_State *J;
727 void *exptr; /* Pointer to exit state. */
728 const BCIns *pc; /* Restart interpreter at this PC. */
729 } ExitDataCP;
730
731 /* Need to protect lj_snap_restore because it may throw. */
732 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
733 {
734 ExitDataCP *exd = (ExitDataCP *)ud;
735 cframe_errfunc(L->cframe) = -1; /* Inherit error function. */
736 exd->pc = lj_snap_restore(exd->J, exd->exptr);
737 UNUSED(dummy);
738 return NULL;
739 }
740
741 #ifndef LUAJIT_DISABLE_VMEVENT
742 /* Push all registers from exit state. */
743 static void trace_exit_regs(lua_State *L, ExitState *ex)
744 {
745 int32_t i;
746 setintV(L->top++, RID_NUM_GPR);
747 setintV(L->top++, RID_NUM_FPR);
748 for (i = 0; i < RID_NUM_GPR; i++) {
749 if (sizeof(ex->gpr[i]) == sizeof(int32_t))
750 setintV(L->top++, (int32_t)ex->gpr[i]);
751 else
752 setnumV(L->top++, (lua_Number)ex->gpr[i]);
753 }
754 #if !LJ_SOFTFP
755 for (i = 0; i < RID_NUM_FPR; i++) {
756 setnumV(L->top, ex->fpr[i]);
757 if (LJ_UNLIKELY(tvisnan(L->top)))
758 setnanV(L->top);
759 L->top++;
760 }
761 #endif
762 }
763 #endif
764
765 #ifdef EXITSTATE_PCREG
766 /* Determine trace number from pc of exit instruction. */
767 static TraceNo trace_exit_find(jit_State *J, MCode *pc)
768 {
769 TraceNo traceno;
770 for (traceno = 1; traceno < J->sizetrace; traceno++) {
771 GCtrace *T = traceref(J, traceno);
772 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
773 return traceno;
774 }
775 lua_assert(0);
776 return 0;
777 }
778 #endif
779
780 /* A trace exited. Restore interpreter state. */
781 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
782 {
783 ERRNO_SAVE
784 lua_State *L = J->L;
785 ExitState *ex = (ExitState *)exptr;
786 ExitDataCP exd;
787 int errcode;
788 const BCIns *pc;
789 void *cf;
790 GCtrace *T;
791 #ifdef EXITSTATE_PCREG
792 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
793 #endif
794 T = traceref(J, J->parent); UNUSED(T);
795 #ifdef EXITSTATE_CHECKEXIT
796 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */
797 lua_assert(T->root != 0);
798 J->exitno = T->ir[REF_BASE].op2;
799 J->parent = T->ir[REF_BASE].op1;
800 T = traceref(J, J->parent);
801 }
802 #endif
803 lua_assert(T != NULL && J->exitno < T->nsnap);
804 exd.J = J;
805 exd.exptr = exptr;
806 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
807 if (errcode)
808 return -errcode; /* Return negated error code. */
809
810 if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
811 lj_vmevent_send(L, TEXIT,
812 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
813 setintV(L->top++, J->parent);
814 setintV(L->top++, J->exitno);
815 trace_exit_regs(L, ex);
816 );
817
818 pc = exd.pc;
819 cf = cframe_raw(L->cframe);
820 setcframe_pc(cf, pc);
821 if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
822 /* Just exit to interpreter. */
823 } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
824 if (!(G(L)->hookmask & HOOK_GC))
825 lj_gc_step(L); /* Exited because of GC: drive GC forward. */
826 } else {
827 trace_hotside(J, pc);
828 }
829 if (bc_op(*pc) == BC_JLOOP) {
830 BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
831 if (bc_isret(bc_op(*retpc))) {
832 if (J->state == LJ_TRACE_RECORD) {
833 J->patchins = *pc;
834 J->patchpc = (BCIns *)pc;
835 *J->patchpc = *retpc;
836 J->bcskip = 1;
837 } else {
838 pc = retpc;
839 setcframe_pc(cf, pc);
840 }
841 }
842 }
843 /* Return MULTRES or 0. */
844 ERRNO_RESTORE
845 switch (bc_op(*pc)) {
846 case BC_CALLM: case BC_CALLMT:
847 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) + LJ_FR2);
848 case BC_RETM:
849 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
850 case BC_TSETM:
851 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
852 default:
853 if (bc_op(*pc) >= BC_FUNCF)
854 return (int)((BCReg)(L->top - L->base) + 1);
855 return 0;
856 }
857 }
858
859 #endif
LuaTeX git mirror