src/lj_trace.c

   1 /*
   2 ** Trace management.
   3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_trace_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT
  12
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_debug.h"
  16 #include "lj_str.h"
  17 #include "lj_frame.h"
  18 #include "lj_state.h"
  19 #include "lj_bc.h"
  20 #include "lj_ir.h"
  21 #include "lj_jit.h"
  22 #include "lj_iropt.h"
  23 #include "lj_mcode.h"
  24 #include "lj_trace.h"
  25 #include "lj_snap.h"
  26 #include "lj_gdbjit.h"
  27 #include "lj_record.h"
  28 #include "lj_asm.h"
  29 #include "lj_dispatch.h"
  30 #include "lj_vm.h"
  31 #include "lj_vmevent.h"
  32 #include "lj_target.h"
  33
  34 /* -- Error handling ------------------------------------------------------ */
  35
  36 /* Synchronous abort with error message. */
  37 void lj_trace_err(jit_State *J, TraceError e)
  38 {
  39   setnilV(&J->errinfo);  /* No error info. */
  40   setintV(J->L->top++, (int32_t)e);
  41   lj_err_throw(J->L, LUA_ERRRUN);
  42 }
  43
  44 /* Synchronous abort with error message and error info. */
  45 void lj_trace_err_info(jit_State *J, TraceError e)
  46 {
  47   setintV(J->L->top++, (int32_t)e);
  48   lj_err_throw(J->L, LUA_ERRRUN);
  49 }
  50
  51 /* -- Trace management ---------------------------------------------------- */
  52
  53 /* The current trace is first assembled in J->cur. The variable length
  54 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
  55 ** recording ends successfully, the current trace and its data structures
  56 ** are copied to a new (compact) GCtrace object.
  57 */
  58
  59 /* Find a free trace number. */
  60 static TraceNo trace_findfree(jit_State *J)
  61 {
  62   MSize osz, lim;
  63   if (J->freetrace == 0)
  64     J->freetrace = 1;
  65   for (; J->freetrace < J->sizetrace; J->freetrace++)
  66     if (traceref(J, J->freetrace) == NULL)
  67       return J->freetrace++;
  68   /* Need to grow trace array. */
  69   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
  70   if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
  71   osz = J->sizetrace;
  72   if (osz >= lim)
  73     return 0;  /* Too many traces. */
  74   lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
  75   for (; osz < J->sizetrace; osz++)
  76     setgcrefnull(J->trace[osz]);
  77   return J->freetrace;
  78 }
  79
  80 #define TRACE_APPENDVEC(field, szfield, tp) \
  81   T->field = (tp *)p; \
  82   memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
  83   p += J->cur.szfield*sizeof(tp);
  84
  85 #ifdef LUAJIT_USE_PERFTOOLS
  86 /*
  87 ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
  88 ** Example usage:
  89 **   perf record -f -e cycles luajit test.lua
  90 **   perf report -s symbol
  91 **   rm perf.data /tmp/perf-*.map
  92 */
  93 #include <stdio.h>
  94 #include <unistd.h>
  95
  96 static void perftools_addtrace(GCtrace *T)
  97 {
  98   static FILE *fp;
  99   GCproto *pt = &gcref(T->startpt)->pt;
 100   const BCIns *startpc = mref(T->startpc, const BCIns);
 101   const char *name = proto_chunknamestr(pt);
 102   BCLine lineno;
 103   if (name[0] == '@' || name[0] == '=')
 104     name++;
 105   else
 106     name = "(string)";
 107   lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
 108   lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
 109   if (!fp) {
 110     char fname[40];
 111     sprintf(fname, "/tmp/perf-%d.map", getpid());
 112     if (!(fp = fopen(fname, "w"))) return;
 113     setlinebuf(fp);
 114   }
 115   fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
 116           (long)T->mcode, T->szmcode, T->traceno, name, lineno);
 117 }
 118 #endif
 119
 120 /* Save current trace by copying and compacting it. */
 121 static void trace_save(jit_State *J)
 122 {
 123   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 124   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
 125   size_t sz = sztr + szins +
 126               J->cur.nsnap*sizeof(SnapShot) +
 127               J->cur.nsnapmap*sizeof(SnapEntry);
 128   GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace);
 129   char *p = (char *)T + sztr;
 130   memcpy(T, &J->cur, sizeof(GCtrace));
 131   setgcrefr(T->nextgc, J2G(J)->gc.root);
 132   setgcrefp(J2G(J)->gc.root, T);
 133   newwhite(J2G(J), T);
 134   T->gct = ~LJ_TTRACE;
 135   T->ir = (IRIns *)p - J->cur.nk;
 136   memcpy(p, J->cur.ir+J->cur.nk, szins);
 137   p += szins;
 138   TRACE_APPENDVEC(snap, nsnap, SnapShot)
 139   TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
 140   J->cur.traceno = 0;
 141   setgcrefp(J->trace[T->traceno], T);
 142   lj_gc_barriertrace(J2G(J), T->traceno);
 143   lj_gdbjit_addtrace(J, T);
 144 #ifdef LUAJIT_USE_PERFTOOLS
 145   perftools_addtrace(T);
 146 #endif
 147 }
 148
 149 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
 150 {
 151   jit_State *J = G2J(g);
 152   if (T->traceno) {
 153     lj_gdbjit_deltrace(J, T);
 154     if (T->traceno < J->freetrace)
 155       J->freetrace = T->traceno;
 156     setgcrefnull(J->trace[T->traceno]);
 157   }
 158   lj_mem_free(g, T,
 159     ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
 160     T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
 161 }
 162
 163 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
 164 void lj_trace_reenableproto(GCproto *pt)
 165 {
 166   if ((pt->flags & PROTO_ILOOP)) {
 167     BCIns *bc = proto_bc(pt);
 168     BCPos i, sizebc = pt->sizebc;;
 169     pt->flags &= ~PROTO_ILOOP;
 170     if (bc_op(bc[0]) == BC_IFUNCF)
 171       setbc_op(&bc[0], BC_FUNCF);
 172     for (i = 1; i < sizebc; i++) {
 173       BCOp op = bc_op(bc[i]);
 174       if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
 175         setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
 176     }
 177   }
 178 }
 179
 180 /* Unpatch the bytecode modified by a root trace. */
 181 static void trace_unpatch(jit_State *J, GCtrace *T)
 182 {
 183   BCOp op = bc_op(T->startins);
 184   BCIns *pc = mref(T->startpc, BCIns);
 185   UNUSED(J);
 186   if (op == BC_JMP)
 187     return;  /* No need to unpatch branches in parent traces (yet). */
 188   switch (bc_op(*pc)) {
 189   case BC_JFORL:
 190     lua_assert(traceref(J, bc_d(*pc)) == T);
 191     *pc = T->startins;
 192     pc += bc_j(T->startins);
 193     lua_assert(bc_op(*pc) == BC_JFORI);
 194     setbc_op(pc, BC_FORI);
 195     break;
 196   case BC_JITERL:
 197   case BC_JLOOP:
 198     lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
 199     *pc = T->startins;
 200     break;
 201   case BC_JMP:
 202     lua_assert(op == BC_ITERL);
 203     pc += bc_j(*pc)+2;
 204     if (bc_op(*pc) == BC_JITERL) {
 205       lua_assert(traceref(J, bc_d(*pc)) == T);
 206       *pc = T->startins;
 207     }
 208     break;
 209   case BC_JFUNCF:
 210     lua_assert(op == BC_FUNCF);
 211     *pc = T->startins;
 212     break;
 213   default:  /* Already unpatched. */
 214     break;
 215   }
 216 }
 217
 218 /* Flush a root trace. */
 219 static void trace_flushroot(jit_State *J, GCtrace *T)
 220 {
 221   GCproto *pt = &gcref(T->startpt)->pt;
 222   lua_assert(T->root == 0 && pt != NULL);
 223   /* First unpatch any modified bytecode. */
 224   trace_unpatch(J, T);
 225   /* Unlink root trace from chain anchored in prototype. */
 226   if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
 227     pt->trace = T->nextroot;
 228   } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
 229     GCtrace *T2 = traceref(J, pt->trace);
 230     if (T2) {
 231       for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
 232         if (T2->nextroot == T->traceno) {
 233           T2->nextroot = T->nextroot;  /* Unlink from chain. */
 234           break;
 235         }
 236     }
 237   }
 238 }
 239
 240 /* Flush a trace. Only root traces are considered. */
 241 void lj_trace_flush(jit_State *J, TraceNo traceno)
 242 {
 243   if (traceno > 0 && traceno < J->sizetrace) {
 244     GCtrace *T = traceref(J, traceno);
 245     if (T && T->root == 0)
 246       trace_flushroot(J, T);
 247   }
 248 }
 249
 250 /* Flush all traces associated with a prototype. */
 251 void lj_trace_flushproto(global_State *g, GCproto *pt)
 252 {
 253   while (pt->trace != 0)
 254     trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
 255 }
 256
 257 /* Flush all traces. */
 258 int lj_trace_flushall(lua_State *L)
 259 {
 260   jit_State *J = L2J(L);
 261   ptrdiff_t i;
 262   if ((J2G(J)->hookmask & HOOK_GC))
 263     return 1;
 264   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
 265     GCtrace *T = traceref(J, i);
 266     if (T) {
 267       if (T->root == 0)
 268         trace_flushroot(J, T);
 269       lj_gdbjit_deltrace(J, T);
 270       T->traceno = 0;
 271       setgcrefnull(J->trace[i]);
 272     }
 273   }
 274   J->cur.traceno = 0;
 275   J->freetrace = 0;
 276   /* Clear penalty cache. */
 277   memset(J->penalty, 0, sizeof(J->penalty));
 278   /* Free the whole machine code and invalidate all exit stub groups. */
 279   lj_mcode_free(J);
 280   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
 281   lj_vmevent_send(L, TRACE,
 282     setstrV(L, L->top++, lj_str_newlit(L, "flush"));
 283   );
 284   return 0;
 285 }
 286
 287 /* Initialize JIT compiler state. */
 288 void lj_trace_initstate(global_State *g)
 289 {
 290   jit_State *J = G2J(g);
 291   TValue *tv;
 292   /* Initialize SIMD constants. */
 293   tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
 294   tv[0].u64 = U64x(7fffffff,ffffffff);
 295   tv[1].u64 = U64x(7fffffff,ffffffff);
 296   tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
 297   tv[0].u64 = U64x(80000000,00000000);
 298   tv[1].u64 = U64x(80000000,00000000);
 299 }
 300
 301 /* Free everything associated with the JIT compiler state. */
 302 void lj_trace_freestate(global_State *g)
 303 {
 304   jit_State *J = G2J(g);
 305 #ifdef LUA_USE_ASSERT
 306   {  /* This assumes all traces have already been freed. */
 307     ptrdiff_t i;
 308     for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
 309       lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
 310   }
 311 #endif
 312   lj_mcode_free(J);
 313   lj_ir_k64_freeall(J);
 314   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
 315   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
 316   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
 317   lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
 318 }
 319
 320 /* -- Penalties and blacklisting ------------------------------------------ */
 321
 322 /* Blacklist a bytecode instruction. */
 323 static void blacklist_pc(GCproto *pt, BCIns *pc)
 324 {
 325   setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
 326   pt->flags |= PROTO_ILOOP;
 327 }
 328
 329 /* Penalize a bytecode instruction. */
 330 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
 331 {
 332   uint32_t i, val = PENALTY_MIN;
 333   for (i = 0; i < PENALTY_SLOTS; i++)
 334     if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
 335       /* First try to bump its hotcount several times. */
 336       val = ((uint32_t)J->penalty[i].val << 1) +
 337             LJ_PRNG_BITS(J, PENALTY_RNDBITS);
 338       if (val > PENALTY_MAX) {
 339         blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
 340         return;
 341       }
 342       goto setpenalty;
 343     }
 344   /* Assign a new penalty cache slot. */
 345   i = J->penaltyslot;
 346   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
 347   setmref(J->penalty[i].pc, pc);
 348 setpenalty:
 349   J->penalty[i].val = (uint16_t)val;
 350   J->penalty[i].reason = e;
 351   hotcount_set(J2GG(J), pc+1, val);
 352 }
 353
 354 /* -- Trace compiler state machine ---------------------------------------- */
 355
 356 /* Start tracing. */
 357 static void trace_start(jit_State *J)
 358 {
 359   lua_State *L;
 360   TraceNo traceno;
 361
 362   if ((J->pt->flags & PROTO_NOJIT)) {  /* JIT disabled for this proto? */
 363     if (J->parent == 0 && J->exitno == 0) {
 364       /* Lazy bytecode patching to disable hotcount events. */
 365       lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
 366                  bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
 367       setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
 368       J->pt->flags |= PROTO_ILOOP;
 369     }
 370     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 371     return;
 372   }
 373
 374   /* Get a new trace number. */
 375   traceno = trace_findfree(J);
 376   if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
 377     lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
 378     lj_trace_flushall(J->L);
 379     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 380     return;
 381   }
 382   setgcrefp(J->trace[traceno], &J->cur);
 383
 384   /* Setup enough of the current trace to be able to send the vmevent. */
 385   memset(&J->cur, 0, sizeof(GCtrace));
 386   J->cur.traceno = traceno;
 387   J->cur.nins = J->cur.nk = REF_BASE;
 388   J->cur.ir = J->irbuf;
 389   J->cur.snap = J->snapbuf;
 390   J->cur.snapmap = J->snapmapbuf;
 391   J->mergesnap = 0;
 392   J->needsnap = 0;
 393   J->bcskip = 0;
 394   J->guardemit.irt = 0;
 395   J->postproc = LJ_POST_NONE;
 396   lj_resetsplit(J);
 397   setgcref(J->cur.startpt, obj2gco(J->pt));
 398
 399   L = J->L;
 400   lj_vmevent_send(L, TRACE,
 401     setstrV(L, L->top++, lj_str_newlit(L, "start"));
 402     setintV(L->top++, traceno);
 403     setfuncV(L, L->top++, J->fn);
 404     setintV(L->top++, proto_bcpos(J->pt, J->pc));
 405     if (J->parent) {
 406       setintV(L->top++, J->parent);
 407       setintV(L->top++, J->exitno);
 408     }
 409   );
 410   lj_record_setup(J);
 411 }
 412
 413 /* Stop tracing. */
 414 static void trace_stop(jit_State *J)
 415 {
 416   BCIns *pc = mref(J->cur.startpc, BCIns);
 417   BCOp op = bc_op(J->cur.startins);
 418   GCproto *pt = &gcref(J->cur.startpt)->pt;
 419   TraceNo traceno = J->cur.traceno;
 420   lua_State *L;
 421
 422   switch (op) {
 423   case BC_FORL:
 424     setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
 425     /* fallthrough */
 426   case BC_LOOP:
 427   case BC_ITERL:
 428   case BC_FUNCF:
 429     /* Patch bytecode of starting instruction in root trace. */
 430     setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
 431     setbc_d(pc, traceno);
 432   addroot:
 433     /* Add to root trace chain in prototype. */
 434     J->cur.nextroot = pt->trace;
 435     pt->trace = (TraceNo1)traceno;
 436     break;
 437   case BC_RET:
 438   case BC_RET0:
 439   case BC_RET1:
 440     *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
 441     goto addroot;
 442   case BC_JMP:
 443     /* Patch exit branch in parent to side trace entry. */
 444     lua_assert(J->parent != 0 && J->cur.root != 0);
 445     lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
 446     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
 447     traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
 448     /* Add to side trace chain in root trace. */
 449     {
 450       GCtrace *root = traceref(J, J->cur.root);
 451       root->nchild++;
 452       J->cur.nextside = root->nextside;
 453       root->nextside = (TraceNo1)traceno;
 454     }
 455     break;
 456   case BC_CALLM:
 457   case BC_CALL:
 458   case BC_ITERC:
 459     /* Trace stitching: patch link of previous trace. */
 460     traceref(J, J->exitno)->link = traceno;
 461     break;
 462   default:
 463     lua_assert(0);
 464     break;
 465   }
 466
 467   /* Commit new mcode only after all patching is done. */
 468   lj_mcode_commit(J, J->cur.mcode);
 469   J->postproc = LJ_POST_NONE;
 470   trace_save(J);
 471
 472   L = J->L;
 473   lj_vmevent_send(L, TRACE,
 474     setstrV(L, L->top++, lj_str_newlit(L, "stop"));
 475     setintV(L->top++, traceno);
 476     setfuncV(L, L->top++, J->fn);
 477   );
 478 }
 479
 480 /* Start a new root trace for down-recursion. */
 481 static int trace_downrec(jit_State *J)
 482 {
 483   /* Restart recording at the return instruction. */
 484   lua_assert(J->pt != NULL);
 485   lua_assert(bc_isret(bc_op(*J->pc)));
 486   if (bc_op(*J->pc) == BC_RETM)
 487     return 0;  /* NYI: down-recursion with RETM. */
 488   J->parent = 0;
 489   J->exitno = 0;
 490   J->state = LJ_TRACE_RECORD;
 491   trace_start(J);
 492   return 1;
 493 }
 494
 495 /* Abort tracing. */
 496 static int trace_abort(jit_State *J)
 497 {
 498   lua_State *L = J->L;
 499   TraceError e = LJ_TRERR_RECERR;
 500   TraceNo traceno;
 501
 502   J->postproc = LJ_POST_NONE;
 503   lj_mcode_abort(J);
 504   if (tvisnumber(L->top-1))
 505     e = (TraceError)numberVint(L->top-1);
 506   if (e == LJ_TRERR_MCODELM) {
 507     L->top--;  /* Remove error object */
 508     J->state = LJ_TRACE_ASM;
 509     return 1;  /* Retry ASM with new MCode area. */
 510   }
 511   /* Penalize or blacklist starting bytecode instruction. */
 512   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
 513     if (J->exitno == 0)
 514       penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e);
 515     else
 516       traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
 517   }
 518
 519   /* Is there anything to abort? */
 520   traceno = J->cur.traceno;
 521   if (traceno) {
 522     ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
 523     J->cur.link = 0;
 524     J->cur.linktype = LJ_TRLINK_NONE;
 525     lj_vmevent_send(L, TRACE,
 526       TValue *frame;
 527       const BCIns *pc;
 528       GCfunc *fn;
 529       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
 530       setintV(L->top++, traceno);
 531       /* Find original Lua function call to generate a better error message. */
 532       frame = J->L->base-1;
 533       pc = J->pc;
 534       while (!isluafunc(frame_func(frame))) {
 535         pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
 536         frame = frame_prev(frame);
 537       }
 538       fn = frame_func(frame);
 539       setfuncV(L, L->top++, fn);
 540       setintV(L->top++, proto_bcpos(funcproto(fn), pc));
 541       copyTV(L, L->top++, restorestack(L, errobj));
 542       copyTV(L, L->top++, &J->errinfo);
 543     );
 544     /* Drop aborted trace after the vmevent (which may still access it). */
 545     setgcrefnull(J->trace[traceno]);
 546     if (traceno < J->freetrace)
 547       J->freetrace = traceno;
 548     J->cur.traceno = 0;
 549   }
 550   L->top--;  /* Remove error object */
 551   if (e == LJ_TRERR_DOWNREC)
 552     return trace_downrec(J);
 553   else if (e == LJ_TRERR_MCODEAL)
 554     lj_trace_flushall(L);
 555   return 0;
 556 }
 557
 558 /* Perform pending re-patch of a bytecode instruction. */
 559 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
 560 {
 561   if (LJ_UNLIKELY(J->patchpc)) {
 562     if (force || J->bcskip == 0) {
 563       *J->patchpc = J->patchins;
 564       J->patchpc = NULL;
 565     } else {
 566       J->bcskip = 0;
 567     }
 568   }
 569 }
 570
 571 /* State machine for the trace compiler. Protected callback. */
 572 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
 573 {
 574   jit_State *J = (jit_State *)ud;
 575   UNUSED(dummy);
 576   do {
 577   retry:
 578     switch (J->state) {
 579     case LJ_TRACE_START:
 580       J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
 581       trace_start(J);
 582       lj_dispatch_update(J2G(J));
 583       break;
 584
 585     case LJ_TRACE_RECORD:
 586       trace_pendpatch(J, 0);
 587       setvmstate(J2G(J), RECORD);
 588       lj_vmevent_send_(L, RECORD,
 589         /* Save/restore tmptv state for trace recorder. */
 590         TValue savetv = J2G(J)->tmptv;
 591         TValue savetv2 = J2G(J)->tmptv2;
 592         setintV(L->top++, J->cur.traceno);
 593         setfuncV(L, L->top++, J->fn);
 594         setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
 595         setintV(L->top++, J->framedepth);
 596       ,
 597         J2G(J)->tmptv = savetv;
 598         J2G(J)->tmptv2 = savetv2;
 599       );
 600       lj_record_ins(J);
 601       break;
 602
 603     case LJ_TRACE_END:
 604       trace_pendpatch(J, 1);
 605       J->loopref = 0;
 606       if ((J->flags & JIT_F_OPT_LOOP) &&
 607           J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 608         setvmstate(J2G(J), OPT);
 609         lj_opt_dce(J);
 610         if (lj_opt_loop(J)) {  /* Loop optimization failed? */
 611           J->cur.link = 0;
 612           J->cur.linktype = LJ_TRLINK_NONE;
 613           J->loopref = J->cur.nins;
 614           J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
 615           break;
 616         }
 617         J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
 618       }
 619       lj_opt_split(J);
 620       lj_opt_sink(J);
 621       if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
 622       J->state = LJ_TRACE_ASM;
 623       break;
 624
 625     case LJ_TRACE_ASM:
 626       setvmstate(J2G(J), ASM);
 627       lj_asm_trace(J, &J->cur);
 628       trace_stop(J);
 629       setvmstate(J2G(J), INTERP);
 630       J->state = LJ_TRACE_IDLE;
 631       lj_dispatch_update(J2G(J));
 632       return NULL;
 633
 634     default:  /* Trace aborted asynchronously. */
 635       setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
 636       /* fallthrough */
 637     case LJ_TRACE_ERR:
 638       trace_pendpatch(J, 1);
 639       if (trace_abort(J))
 640         goto retry;
 641       setvmstate(J2G(J), INTERP);
 642       J->state = LJ_TRACE_IDLE;
 643       lj_dispatch_update(J2G(J));
 644       return NULL;
 645     }
 646   } while (J->state > LJ_TRACE_RECORD);
 647   return NULL;
 648 }
 649
 650 /* -- Event handling ------------------------------------------------------ */
 651
 652 /* A bytecode instruction is about to be executed. Record it. */
 653 void lj_trace_ins(jit_State *J, const BCIns *pc)
 654 {
 655   /* Note: J->L must already be set. pc is the true bytecode PC here. */
 656   J->pc = pc;
 657   J->fn = curr_func(J->L);
 658   J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
 659   while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
 660     J->state = LJ_TRACE_ERR;
 661 }
 662
 663 /* A hotcount triggered. Start recording a root trace. */
 664 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 665 {
 666   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
 667   ERRNO_SAVE
 668   /* Reset hotcount. */
 669   hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
 670   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 671   if (J->state == LJ_TRACE_IDLE &&
 672       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 673     J->parent = 0;  /* Root trace. */
 674     J->exitno = 0;
 675     J->state = LJ_TRACE_START;
 676     lj_trace_ins(J, pc-1);
 677   }
 678   ERRNO_RESTORE
 679 }
 680
 681 /* Check for a hot side exit. If yes, start recording a side trace. */
 682 static void trace_hotside(jit_State *J, const BCIns *pc)
 683 {
 684   SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
 685   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
 686       isluafunc(curr_func(J->L)) &&
 687       snap->count != SNAPCOUNT_DONE &&
 688       ++snap->count >= J->param[JIT_P_hotexit]) {
 689     lua_assert(J->state == LJ_TRACE_IDLE);
 690     /* J->parent is non-zero for a side trace. */
 691     J->state = LJ_TRACE_START;
 692     lj_trace_ins(J, pc);
 693   }
 694 }
 695
 696 /* Stitch a new trace to the previous trace. */
 697 void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
 698 {
 699   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 700   if (J->state == LJ_TRACE_IDLE &&
 701       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 702     J->parent = 0;  /* Have to treat it like a root trace. */
 703     /* J->exitno is set to the invoking trace. */
 704     J->state = LJ_TRACE_START;
 705     lj_trace_ins(J, pc);
 706   }
 707 }
 708
 709
 710 /* Tiny struct to pass data to protected call. */
 711 typedef struct ExitDataCP {
 712   jit_State *J;
 713   void *exptr;          /* Pointer to exit state. */
 714   const BCIns *pc;      /* Restart interpreter at this PC. */
 715 } ExitDataCP;
 716
 717 /* Need to protect lj_snap_restore because it may throw. */
 718 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
 719 {
 720   ExitDataCP *exd = (ExitDataCP *)ud;
 721   cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
 722   exd->pc = lj_snap_restore(exd->J, exd->exptr);
 723   UNUSED(dummy);
 724   return NULL;
 725 }
 726
 727 #ifndef LUAJIT_DISABLE_VMEVENT
 728 /* Push all registers from exit state. */
 729 static void trace_exit_regs(lua_State *L, ExitState *ex)
 730 {
 731   int32_t i;
 732   setintV(L->top++, RID_NUM_GPR);
 733   setintV(L->top++, RID_NUM_FPR);
 734   for (i = 0; i < RID_NUM_GPR; i++) {
 735     if (sizeof(ex->gpr[i]) == sizeof(int32_t))
 736       setintV(L->top++, (int32_t)ex->gpr[i]);
 737     else
 738       setnumV(L->top++, (lua_Number)ex->gpr[i]);
 739   }
 740 #if !LJ_SOFTFP
 741   for (i = 0; i < RID_NUM_FPR; i++) {
 742     setnumV(L->top, ex->fpr[i]);
 743     if (LJ_UNLIKELY(tvisnan(L->top)))
 744       setnanV(L->top);
 745     L->top++;
 746   }
 747 #endif
 748 }
 749 #endif
 750
 751 #ifdef EXITSTATE_PCREG
 752 /* Determine trace number from pc of exit instruction. */
 753 static TraceNo trace_exit_find(jit_State *J, MCode *pc)
 754 {
 755   TraceNo traceno;
 756   for (traceno = 1; traceno < J->sizetrace; traceno++) {
 757     GCtrace *T = traceref(J, traceno);
 758     if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
 759       return traceno;
 760   }
 761   lua_assert(0);
 762   return 0;
 763 }
 764 #endif
 765
 766 /* A trace exited. Restore interpreter state. */
 767 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
 768 {
 769   ERRNO_SAVE
 770   lua_State *L = J->L;
 771   ExitState *ex = (ExitState *)exptr;
 772   ExitDataCP exd;
 773   int errcode;
 774   const BCIns *pc;
 775   void *cf;
 776   GCtrace *T;
 777 #ifdef EXITSTATE_PCREG
 778   J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
 779 #endif
 780   T = traceref(J, J->parent); UNUSED(T);
 781 #ifdef EXITSTATE_CHECKEXIT
 782   if (J->exitno == T->nsnap) {  /* Treat stack check like a parent exit. */
 783     lua_assert(T->root != 0);
 784     J->exitno = T->ir[REF_BASE].op2;
 785     J->parent = T->ir[REF_BASE].op1;
 786     T = traceref(J, J->parent);
 787   }
 788 #endif
 789   lua_assert(T != NULL && J->exitno < T->nsnap);
 790   exd.J = J;
 791   exd.exptr = exptr;
 792   errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
 793   if (errcode)
 794     return -errcode;  /* Return negated error code. */
 795
 796   if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
 797     lj_vmevent_send(L, TEXIT,
 798       lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
 799       setintV(L->top++, J->parent);
 800       setintV(L->top++, J->exitno);
 801       trace_exit_regs(L, ex);
 802     );
 803
 804   pc = exd.pc;
 805   cf = cframe_raw(L->cframe);
 806   setcframe_pc(cf, pc);
 807   if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
 808     /* Just exit to interpreter. */
 809   } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
 810     if (!(G(L)->hookmask & HOOK_GC))
 811       lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
 812   } else {
 813     trace_hotside(J, pc);
 814   }
 815   if (bc_op(*pc) == BC_JLOOP) {
 816     BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
 817     if (bc_isret(bc_op(*retpc))) {
 818       if (J->state == LJ_TRACE_RECORD) {
 819         J->patchins = *pc;
 820         J->patchpc = (BCIns *)pc;
 821         *J->patchpc = *retpc;
 822         J->bcskip = 1;
 823       } else {
 824         pc = retpc;
 825         setcframe_pc(cf, pc);
 826       }
 827     }
 828   }
 829   /* Return MULTRES or 0. */
 830   ERRNO_RESTORE
 831   switch (bc_op(*pc)) {
 832   case BC_CALLM: case BC_CALLMT:
 833     return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc));
 834   case BC_RETM:
 835     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
 836   case BC_TSETM:
 837     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
 838   default:
 839     if (bc_op(*pc) >= BC_FUNCF)
 840       return (int)((BCReg)(L->top - L->base) + 1);
 841     return 0;
 842   }
 843 }
 844
 845 #endif