src/lj_trace.c

   1 /*
   2 ** Trace management.
   3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_trace_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT
  12
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_str.h"
  16 #include "lj_frame.h"
  17 #include "lj_state.h"
  18 #include "lj_bc.h"
  19 #include "lj_ir.h"
  20 #include "lj_jit.h"
  21 #include "lj_iropt.h"
  22 #include "lj_mcode.h"
  23 #include "lj_trace.h"
  24 #include "lj_snap.h"
  25 #include "lj_gdbjit.h"
  26 #include "lj_record.h"
  27 #include "lj_asm.h"
  28 #include "lj_dispatch.h"
  29 #include "lj_vm.h"
  30 #include "lj_vmevent.h"
  31 #include "lj_target.h"
  32
  33 /* -- Error handling ------------------------------------------------------ */
  34
  35 /* Synchronous abort with error message. */
  36 void lj_trace_err(jit_State *J, TraceError e)
  37 {
  38   setnilV(&J->errinfo);  /* No error info. */
  39   setintV(J->L->top++, (int32_t)e);
  40   lj_err_throw(J->L, LUA_ERRRUN);
  41 }
  42
  43 /* Synchronous abort with error message and error info. */
  44 void lj_trace_err_info(jit_State *J, TraceError e)
  45 {
  46   setintV(J->L->top++, (int32_t)e);
  47   lj_err_throw(J->L, LUA_ERRRUN);
  48 }
  49
  50 /* -- Trace management ---------------------------------------------------- */
  51
  52 /* The current trace is first assembled in J->cur. The variable length
  53 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
  54 ** recording ends successfully, the current trace and its data structures
  55 ** are copied to a new (compact) GCtrace object.
  56 */
  57
  58 /* Find a free trace number. */
  59 static TraceNo trace_findfree(jit_State *J)
  60 {
  61   MSize osz, lim;
  62   if (J->freetrace == 0)
  63     J->freetrace = 1;
  64   for (; J->freetrace < J->sizetrace; J->freetrace++)
  65     if (traceref(J, J->freetrace) == NULL)
  66       return J->freetrace++;
  67   /* Need to grow trace array. */
  68   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
  69   if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
  70   osz = J->sizetrace;
  71   if (osz >= lim)
  72     return 0;  /* Too many traces. */
  73   lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
  74   for (; osz < J->sizetrace; osz++)
  75     setgcrefnull(J->trace[osz]);
  76   return J->freetrace;
  77 }
  78
  79 #define TRACE_APPENDVEC(field, szfield, tp) \
  80   T->field = (tp *)p; \
  81   memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
  82   p += J->cur.szfield*sizeof(tp);
  83
  84 #ifdef LUAJIT_USE_PERFTOOLS
  85 /*
  86 ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
  87 ** Example usage:
  88 **   perf record -f -e cycles luajit test.lua
  89 **   perf report -s symbol
  90 **   rm perf.data /tmp/perf-*.map
  91 */
  92 #include <stdio.h>
  93 #include <unistd.h>
  94
  95 static void perftools_addtrace(GCtrace *T)
  96 {
  97   static FILE *fp;
  98   GCproto *pt = &gcref(T->startpt)->pt;
  99   const BCIns *startpc = mref(T->startpc, const BCIns);
 100   const char *name = strdata(proto_chunkname(pt));
 101   BCLine lineno;
 102   if (name[0] == '@' || name[0] == '=')
 103     name++;
 104   else
 105     name = "(string)";
 106   if (startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc)
 107     lineno = proto_line(pt, proto_bcpos(pt, startpc));
 108   else
 109     lineno = proto_line(pt, 0);  /* Wrong, but better than nothing. */
 110   if (!fp) {
 111     char fname[40];
 112     sprintf(fname, "/tmp/perf-%d.map", getpid());
 113     if (!(fp = fopen(fname, "w"))) return;
 114     setlinebuf(fp);
 115   }
 116   fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
 117           (long)T->mcode, T->szmcode, T->traceno, name, lineno);
 118 }
 119 #endif
 120
 121 /* Save current trace by copying and compacting it. */
 122 static void trace_save(jit_State *J)
 123 {
 124   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 125   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
 126   size_t sz = sztr + szins +
 127               J->cur.nsnap*sizeof(SnapShot) +
 128               J->cur.nsnapmap*sizeof(SnapEntry);
 129   GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace);
 130   char *p = (char *)T + sztr;
 131   memcpy(T, &J->cur, sizeof(GCtrace));
 132   setgcrefr(T->nextgc, J2G(J)->gc.root);
 133   setgcrefp(J2G(J)->gc.root, T);
 134   newwhite(J2G(J), T);
 135   T->gct = ~LJ_TTRACE;
 136   T->ir = (IRIns *)p - J->cur.nk;
 137   memcpy(p, J->cur.ir+J->cur.nk, szins);
 138   p += szins;
 139   TRACE_APPENDVEC(snap, nsnap, SnapShot)
 140   TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
 141   J->cur.traceno = 0;
 142   setgcrefp(J->trace[T->traceno], T);
 143   lj_gc_barriertrace(J2G(J), T->traceno);
 144   lj_gdbjit_addtrace(J, T);
 145 #ifdef LUAJIT_USE_PERFTOOLS
 146   perftools_addtrace(T);
 147 #endif
 148 }
 149
 150 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
 151 {
 152   jit_State *J = G2J(g);
 153   if (T->traceno) {
 154     lj_gdbjit_deltrace(J, T);
 155     if (T->traceno < J->freetrace)
 156       J->freetrace = T->traceno;
 157     setgcrefnull(J->trace[T->traceno]);
 158   }
 159   lj_mem_free(g, T,
 160     ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
 161     T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
 162 }
 163
 164 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
 165 void lj_trace_reenableproto(GCproto *pt)
 166 {
 167   if ((pt->flags & PROTO_HAS_ILOOP)) {
 168     BCIns *bc = proto_bc(pt);
 169     BCPos i, sizebc = pt->sizebc;;
 170     pt->flags &= ~PROTO_HAS_ILOOP;
 171     if (bc_op(bc[0]) == BC_IFUNCF)
 172       setbc_op(&bc[0], BC_FUNCF);
 173     for (i = 1; i < sizebc; i++) {
 174       BCOp op = bc_op(bc[i]);
 175       if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
 176         setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
 177     }
 178   }
 179 }
 180
 181 /* Unpatch the bytecode modified by a root trace. */
 182 static void trace_unpatch(jit_State *J, GCtrace *T)
 183 {
 184   BCOp op = bc_op(T->startins);
 185   BCIns *pc = mref(T->startpc, BCIns);
 186   UNUSED(J);
 187   if (op == BC_JMP)
 188     return;  /* No need to unpatch branches in parent traces (yet). */
 189   switch (bc_op(*pc)) {
 190   case BC_JFORL:
 191     lua_assert(traceref(J, bc_d(*pc)) == T);
 192     *pc = T->startins;
 193     pc += bc_j(T->startins);
 194     lua_assert(bc_op(*pc) == BC_JFORI);
 195     setbc_op(pc, BC_FORI);
 196     break;
 197   case BC_JITERL:
 198   case BC_JLOOP:
 199     lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
 200     *pc = T->startins;
 201     break;
 202   case BC_JMP:
 203     lua_assert(op == BC_ITERL);
 204     pc += bc_j(*pc)+2;
 205     if (bc_op(*pc) == BC_JITERL) {
 206       lua_assert(traceref(J, bc_d(*pc)) == T);
 207       *pc = T->startins;
 208     }
 209     break;
 210   case BC_JFUNCF:
 211     lua_assert(op == BC_FUNCF);
 212     *pc = T->startins;
 213     break;
 214   default:  /* Already unpatched. */
 215     break;
 216   }
 217 }
 218
 219 /* Flush a root trace. */
 220 static void trace_flushroot(jit_State *J, GCtrace *T)
 221 {
 222   GCproto *pt = &gcref(T->startpt)->pt;
 223   lua_assert(T->root == 0 && pt != NULL);
 224   /* First unpatch any modified bytecode. */
 225   trace_unpatch(J, T);
 226   /* Unlink root trace from chain anchored in prototype. */
 227   if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
 228     pt->trace = T->nextroot;
 229   } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
 230     GCtrace *T2 = traceref(J, pt->trace);
 231     if (T2) {
 232       for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
 233         if (T2->nextroot == T->traceno) {
 234           T2->nextroot = T->nextroot;  /* Unlink from chain. */
 235           break;
 236         }
 237     }
 238   }
 239 }
 240
 241 /* Flush a trace. Only root traces are considered. */
 242 void lj_trace_flush(jit_State *J, TraceNo traceno)
 243 {
 244   if (traceno > 0 && traceno < J->sizetrace) {
 245     GCtrace *T = traceref(J, traceno);
 246     if (T && T->root == 0)
 247       trace_flushroot(J, T);
 248   }
 249 }
 250
 251 /* Flush all traces associated with a prototype. */
 252 void lj_trace_flushproto(global_State *g, GCproto *pt)
 253 {
 254   while (pt->trace != 0)
 255     trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
 256 }
 257
 258 /* Flush all traces. */
 259 int lj_trace_flushall(lua_State *L)
 260 {
 261   jit_State *J = L2J(L);
 262   ptrdiff_t i;
 263   if ((J2G(J)->hookmask & HOOK_GC))
 264     return 1;
 265   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
 266     GCtrace *T = traceref(J, i);
 267     if (T) {
 268       if (T->root == 0)
 269         trace_flushroot(J, T);
 270       lj_gdbjit_deltrace(J, T);
 271       T->traceno = 0;
 272       setgcrefnull(J->trace[i]);
 273     }
 274   }
 275   J->cur.traceno = 0;
 276   J->freetrace = 0;
 277   /* Clear penalty cache. */
 278   memset(J->penalty, 0, sizeof(J->penalty));
 279   /* Free the whole machine code and invalidate all exit stub groups. */
 280   lj_mcode_free(J);
 281   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
 282   lj_vmevent_send(L, TRACE,
 283     setstrV(L, L->top++, lj_str_newlit(L, "flush"));
 284   );
 285   return 0;
 286 }
 287
 288 /* Initialize JIT compiler state. */
 289 void lj_trace_initstate(global_State *g)
 290 {
 291   jit_State *J = G2J(g);
 292   TValue *tv;
 293   /* Initialize SIMD constants. */
 294   tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
 295   tv[0].u64 = U64x(7fffffff,ffffffff);
 296   tv[1].u64 = U64x(7fffffff,ffffffff);
 297   tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
 298   tv[0].u64 = U64x(80000000,00000000);
 299   tv[1].u64 = U64x(80000000,00000000);
 300 }
 301
 302 /* Free everything associated with the JIT compiler state. */
 303 void lj_trace_freestate(global_State *g)
 304 {
 305   jit_State *J = G2J(g);
 306 #ifdef LUA_USE_ASSERT
 307   {  /* This assumes all traces have already been freed. */
 308     ptrdiff_t i;
 309     for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
 310       lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
 311   }
 312 #endif
 313   lj_mcode_free(J);
 314   lj_ir_k64_freeall(J);
 315   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
 316   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
 317   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
 318   lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
 319 }
 320
 321 /* -- Penalties and blacklisting ------------------------------------------ */
 322
 323 /* Blacklist a bytecode instruction. */
 324 static void blacklist_pc(GCproto *pt, BCIns *pc)
 325 {
 326   setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
 327   pt->flags |= PROTO_HAS_ILOOP;
 328 }
 329
 330 /* Penalize a bytecode instruction. */
 331 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
 332 {
 333   uint32_t i, val = PENALTY_MIN;
 334   for (i = 0; i < PENALTY_SLOTS; i++)
 335     if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
 336       /* First try to bump its hotcount several times. */
 337       val = ((uint32_t)J->penalty[i].val << 1) +
 338             LJ_PRNG_BITS(J, PENALTY_RNDBITS);
 339       if (val > PENALTY_MAX) {
 340         blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
 341         return;
 342       }
 343       goto setpenalty;
 344     }
 345   /* Assign a new penalty cache slot. */
 346   i = J->penaltyslot;
 347   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
 348   setmref(J->penalty[i].pc, pc);
 349 setpenalty:
 350   J->penalty[i].val = (uint16_t)val;
 351   J->penalty[i].reason = e;
 352   hotcount_set(J2GG(J), pc+1, val);
 353 }
 354
 355 /* -- Trace compiler state machine ---------------------------------------- */
 356
 357 /* Start tracing. */
 358 static void trace_start(jit_State *J)
 359 {
 360   lua_State *L;
 361   TraceNo traceno;
 362
 363   if ((J->pt->flags & PROTO_NO_JIT)) {  /* JIT disabled for this proto? */
 364     if (J->parent == 0) {
 365       /* Lazy bytecode patching to disable hotcount events. */
 366       lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
 367                  bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
 368       setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
 369       J->pt->flags |= PROTO_HAS_ILOOP;
 370     }
 371     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 372     return;
 373   }
 374
 375   /* Get a new trace number. */
 376   traceno = trace_findfree(J);
 377   if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
 378     lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
 379     lj_trace_flushall(J->L);
 380     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 381     return;
 382   }
 383   setgcrefp(J->trace[traceno], &J->cur);
 384
 385   /* Setup enough of the current trace to be able to send the vmevent. */
 386   memset(&J->cur, 0, sizeof(GCtrace));
 387   J->cur.traceno = traceno;
 388   J->cur.nins = J->cur.nk = REF_BASE;
 389   J->cur.ir = J->irbuf;
 390   J->cur.snap = J->snapbuf;
 391   J->cur.snapmap = J->snapmapbuf;
 392   J->mergesnap = 0;
 393   J->needsnap = 0;
 394   J->bcskip = 0;
 395   J->guardemit.irt = 0;
 396   J->postproc = LJ_POST_NONE;
 397   lj_resetsplit(J);
 398   setgcref(J->cur.startpt, obj2gco(J->pt));
 399
 400   L = J->L;
 401   lj_vmevent_send(L, TRACE,
 402     setstrV(L, L->top++, lj_str_newlit(L, "start"));
 403     setintV(L->top++, traceno);
 404     setfuncV(L, L->top++, J->fn);
 405     setintV(L->top++, proto_bcpos(J->pt, J->pc));
 406     if (J->parent) {
 407       setintV(L->top++, J->parent);
 408       setintV(L->top++, J->exitno);
 409     }
 410   );
 411   lj_record_setup(J);
 412 }
 413
 414 /* Stop tracing. */
 415 static void trace_stop(jit_State *J)
 416 {
 417   BCIns *pc = mref(J->cur.startpc, BCIns);
 418   BCOp op = bc_op(J->cur.startins);
 419   GCproto *pt = &gcref(J->cur.startpt)->pt;
 420   TraceNo traceno = J->cur.traceno;
 421   lua_State *L;
 422
 423   switch (op) {
 424   case BC_FORL:
 425     setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
 426     /* fallthrough */
 427   case BC_LOOP:
 428   case BC_ITERL:
 429   case BC_FUNCF:
 430     /* Patch bytecode of starting instruction in root trace. */
 431     setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
 432     setbc_d(pc, traceno);
 433   addroot:
 434     /* Add to root trace chain in prototype. */
 435     J->cur.nextroot = pt->trace;
 436     pt->trace = (TraceNo1)traceno;
 437     break;
 438   case BC_RET:
 439   case BC_RET0:
 440   case BC_RET1:
 441     *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
 442     goto addroot;
 443   case BC_JMP:
 444     /* Patch exit branch in parent to side trace entry. */
 445     lua_assert(J->parent != 0 && J->cur.root != 0);
 446     lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
 447     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
 448     traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
 449     /* Add to side trace chain in root trace. */
 450     {
 451       GCtrace *root = traceref(J, J->cur.root);
 452       root->nchild++;
 453       J->cur.nextside = root->nextside;
 454       root->nextside = (TraceNo1)traceno;
 455     }
 456     break;
 457   default:
 458     lua_assert(0);
 459     break;
 460   }
 461
 462   /* Commit new mcode only after all patching is done. */
 463   lj_mcode_commit(J, J->cur.mcode);
 464   J->postproc = LJ_POST_NONE;
 465   trace_save(J);
 466
 467   L = J->L;
 468   lj_vmevent_send(L, TRACE,
 469     setstrV(L, L->top++, lj_str_newlit(L, "stop"));
 470     setintV(L->top++, traceno);
 471   );
 472 }
 473
 474 /* Start a new root trace for down-recursion. */
 475 static int trace_downrec(jit_State *J)
 476 {
 477   /* Restart recording at the return instruction. */
 478   lua_assert(J->pt != NULL);
 479   lua_assert(bc_isret(bc_op(*J->pc)));
 480   if (bc_op(*J->pc) == BC_RETM)
 481     return 0;  /* NYI: down-recursion with RETM. */
 482   J->parent = 0;
 483   J->exitno = 0;
 484   J->state = LJ_TRACE_RECORD;
 485   trace_start(J);
 486   return 1;
 487 }
 488
 489 /* Abort tracing. */
 490 static int trace_abort(jit_State *J)
 491 {
 492   lua_State *L = J->L;
 493   TraceError e = LJ_TRERR_RECERR;
 494   TraceNo traceno;
 495
 496   J->postproc = LJ_POST_NONE;
 497   lj_mcode_abort(J);
 498   if (tvisnum(L->top-1))
 499     e = (TraceError)lj_num2int(numV(L->top-1));
 500   if (e == LJ_TRERR_MCODELM) {
 501     J->state = LJ_TRACE_ASM;
 502     return 1;  /* Retry ASM with new MCode area. */
 503   }
 504   /* Penalize or blacklist starting bytecode instruction. */
 505   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins)))
 506     penalty_pc(J, &gcref(J->cur.startpt)->pt, (BCIns *)J->startpc, e);
 507
 508   /* Is there anything to abort? */
 509   traceno = J->cur.traceno;
 510   if (traceno) {
 511     ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
 512     J->cur.link = 0;
 513     lj_vmevent_send(L, TRACE,
 514       TValue *frame;
 515       const BCIns *pc;
 516       GCfunc *fn;
 517       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
 518       setintV(L->top++, traceno);
 519       /* Find original Lua function call to generate a better error message. */
 520       frame = J->L->base-1;
 521       pc = J->pc;
 522       while (!isluafunc(frame_func(frame))) {
 523         pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
 524         frame = frame_prev(frame);
 525       }
 526       fn = frame_func(frame);
 527       setfuncV(L, L->top++, fn);
 528       setintV(L->top++, proto_bcpos(funcproto(fn), pc));
 529       copyTV(L, L->top++, restorestack(L, errobj));
 530       copyTV(L, L->top++, &J->errinfo);
 531     );
 532     /* Drop aborted trace after the vmevent (which may still access it). */
 533     setgcrefnull(J->trace[traceno]);
 534     if (traceno < J->freetrace)
 535       J->freetrace = traceno;
 536     J->cur.traceno = 0;
 537   }
 538   L->top--;  /* Remove error object */
 539   if (e == LJ_TRERR_DOWNREC)
 540     return trace_downrec(J);
 541   else if (e == LJ_TRERR_MCODEAL)
 542     lj_trace_flushall(L);
 543   return 0;
 544 }
 545
 546 /* Perform pending re-patch of a bytecode instruction. */
 547 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
 548 {
 549   if (LJ_UNLIKELY(J->patchpc) && (force || J->chain[IR_RETF])) {
 550     *J->patchpc = J->patchins;
 551     J->patchpc = NULL;
 552   }
 553 }
 554
 555 /* State machine for the trace compiler. Protected callback. */
 556 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
 557 {
 558   jit_State *J = (jit_State *)ud;
 559   UNUSED(dummy);
 560   do {
 561   retry:
 562     switch (J->state) {
 563     case LJ_TRACE_START:
 564       J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
 565       trace_start(J);
 566       lj_dispatch_update(J2G(J));
 567       break;
 568
 569     case LJ_TRACE_RECORD:
 570       trace_pendpatch(J, 0);
 571       setvmstate(J2G(J), RECORD);
 572       lj_vmevent_send(L, RECORD,
 573         setintV(L->top++, J->cur.traceno);
 574         setfuncV(L, L->top++, J->fn);
 575         setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
 576         setintV(L->top++, J->framedepth);
 577       );
 578       lj_record_ins(J);
 579       break;
 580
 581     case LJ_TRACE_END:
 582       trace_pendpatch(J, 1);
 583       J->loopref = 0;
 584       if ((J->flags & JIT_F_OPT_LOOP) &&
 585           J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 586         setvmstate(J2G(J), OPT);
 587         lj_opt_dce(J);
 588         if (lj_opt_loop(J)) {  /* Loop optimization failed? */
 589           J->cur.link = 0;
 590           J->loopref = J->cur.nins;
 591           J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
 592           break;
 593         }
 594         J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
 595       }
 596       lj_opt_split(J);
 597       J->state = LJ_TRACE_ASM;
 598       break;
 599
 600     case LJ_TRACE_ASM:
 601       setvmstate(J2G(J), ASM);
 602       lj_asm_trace(J, &J->cur);
 603       trace_stop(J);
 604       setvmstate(J2G(J), INTERP);
 605       J->state = LJ_TRACE_IDLE;
 606       lj_dispatch_update(J2G(J));
 607       return NULL;
 608
 609     default:  /* Trace aborted asynchronously. */
 610       setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
 611       /* fallthrough */
 612     case LJ_TRACE_ERR:
 613       trace_pendpatch(J, 1);
 614       if (trace_abort(J))
 615         goto retry;
 616       setvmstate(J2G(J), INTERP);
 617       J->state = LJ_TRACE_IDLE;
 618       lj_dispatch_update(J2G(J));
 619       return NULL;
 620     }
 621   } while (J->state > LJ_TRACE_RECORD);
 622   return NULL;
 623 }
 624
 625 /* -- Event handling ------------------------------------------------------ */
 626
 627 /* A bytecode instruction is about to be executed. Record it. */
 628 void lj_trace_ins(jit_State *J, const BCIns *pc)
 629 {
 630   /* Note: J->L must already be set. pc is the true bytecode PC here. */
 631   J->pc = pc;
 632   J->fn = curr_func(J->L);
 633   J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
 634   while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
 635     J->state = LJ_TRACE_ERR;
 636 }
 637
 638 /* A hotcount triggered. Start recording a root trace. */
 639 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 640 {
 641   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
 642   hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]+1);  /* Reset hotcount. */
 643   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 644   if (J->state == LJ_TRACE_IDLE &&
 645       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 646     J->parent = 0;  /* Root trace. */
 647     J->exitno = 0;
 648     J->state = LJ_TRACE_START;
 649     lj_trace_ins(J, pc-1);
 650   }
 651 }
 652
 653 /* Check for a hot side exit. If yes, start recording a side trace. */
 654 static void trace_hotside(jit_State *J, const BCIns *pc)
 655 {
 656   SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
 657   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
 658       snap->count != SNAPCOUNT_DONE &&
 659       ++snap->count >= J->param[JIT_P_hotexit]) {
 660     lua_assert(J->state == LJ_TRACE_IDLE);
 661     /* J->parent is non-zero for a side trace. */
 662     J->state = LJ_TRACE_START;
 663     lj_trace_ins(J, pc);
 664   }
 665 }
 666
 667 /* Tiny struct to pass data to protected call. */
 668 typedef struct ExitDataCP {
 669   jit_State *J;
 670   void *exptr;          /* Pointer to exit state. */
 671   const BCIns *pc;      /* Restart interpreter at this PC. */
 672 } ExitDataCP;
 673
 674 /* Need to protect lj_snap_restore because it may throw. */
 675 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
 676 {
 677   ExitDataCP *exd = (ExitDataCP *)ud;
 678   cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
 679   exd->pc = lj_snap_restore(exd->J, exd->exptr);
 680   UNUSED(dummy);
 681   return NULL;
 682 }
 683
 684 /* A trace exited. Restore interpreter state. */
 685 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
 686 {
 687   lua_State *L = J->L;
 688   ExitDataCP exd;
 689   int errcode;
 690   const BCIns *pc;
 691   void *cf;
 692   exd.J = J;
 693   exd.exptr = exptr;
 694   errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
 695   if (errcode)
 696     return -errcode;  /* Return negated error code. */
 697
 698   lj_vmevent_send(L, TEXIT,
 699     ExitState *ex = (ExitState *)exptr;
 700     uint32_t i;
 701     lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
 702     setintV(L->top++, J->parent);
 703     setintV(L->top++, J->exitno);
 704     setintV(L->top++, RID_NUM_GPR);
 705     setintV(L->top++, RID_NUM_FPR);
 706     for (i = 0; i < RID_NUM_GPR; i++)
 707       setnumV(L->top++, cast_num(ex->gpr[i]));
 708     for (i = 0; i < RID_NUM_FPR; i++) {
 709       setnumV(L->top, ex->fpr[i]);
 710       if (LJ_UNLIKELY(tvisnan(L->top)))
 711         setnanV(L->top);
 712       L->top++;
 713     }
 714   );
 715
 716   pc = exd.pc;
 717   cf = cframe_raw(L->cframe);
 718   setcframe_pc(cf, pc);
 719   if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize)
 720     lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
 721   else
 722     trace_hotside(J, pc);
 723   if (bc_op(*pc) == BC_JLOOP) {
 724     BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
 725     if (bc_isret(bc_op(*retpc))) {
 726       if (J->state == LJ_TRACE_RECORD) {
 727         J->patchins = *pc;
 728         J->patchpc = (BCIns *)pc;
 729         *J->patchpc = *retpc;
 730       } else {
 731         pc = retpc;
 732         setcframe_pc(cf, pc);
 733       }
 734     }
 735   }
 736   /* Return MULTRES or 0. */
 737   switch (bc_op(*pc)) {
 738   case BC_CALLM: case BC_CALLMT:
 739     return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc));
 740   case BC_RETM:
 741     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
 742   case BC_TSETM:
 743     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
 744   default:
 745     if (bc_op(*pc) >= BC_FUNCF)
 746       return (int)((BCReg)(L->top - L->base) + 1);
 747     return 0;
 748   }
 749 }
 750
 751 #endif