source/libs/luajit/LuaJIT-src/src/lj_dispatch.c

   1 /*
   2 ** Instruction dispatch handling.
   3 ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_dispatch_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10 #include "lj_err.h"
  11 #include "lj_buf.h"
  12 #include "lj_func.h"
  13 #include "lj_str.h"
  14 #include "lj_tab.h"
  15 #include "lj_meta.h"
  16 #include "lj_debug.h"
  17 #include "lj_state.h"
  18 #include "lj_frame.h"
  19 #include "lj_bc.h"
  20 #include "lj_ff.h"
  21 #include "lj_strfmt.h"
  22 #if LJ_HASJIT
  23 #include "lj_jit.h"
  24 #endif
  25 #if LJ_HASFFI
  26 #include "lj_ccallback.h"
  27 #endif
  28 #include "lj_trace.h"
  29 #include "lj_dispatch.h"
  30 #if LJ_HASPROFILE
  31 #include "lj_profile.h"
  32 #endif
  33 #include "lj_vm.h"
  34 #include "luajit.h"
  35
  36 /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
  37 LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
  38
  39 /* -- Dispatch table management ------------------------------------------- */
  40
  41 #if LJ_TARGET_MIPS
  42 #include <math.h>
  43 LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
  44                                                           lua_State *co);
  45 #if !LJ_HASJIT
  46 #define lj_dispatch_stitch      lj_dispatch_ins
  47 #endif
  48 #if !LJ_HASPROFILE
  49 #define lj_dispatch_profile     lj_dispatch_ins
  50 #endif
  51
  52 #define GOTFUNC(name)   (ASMFunction)name,
  53 static const ASMFunction dispatch_got[] = {
  54   GOTDEF(GOTFUNC)
  55 };
  56 #undef GOTFUNC
  57 #endif
  58
  59 /* Initialize instruction dispatch table and hot counters. */
  60 void lj_dispatch_init(GG_State *GG)
  61 {
  62   uint32_t i;
  63   ASMFunction *disp = GG->dispatch;
  64   for (i = 0; i < GG_LEN_SDISP; i++)
  65     disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  66   for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  67     disp[i] = makeasmfunc(lj_bc_ofs[i]);
  68   /* The JIT engine is off by default. luaopen_jit() turns it on. */
  69   disp[BC_FORL] = disp[BC_IFORL];
  70   disp[BC_ITERL] = disp[BC_IITERL];
  71   disp[BC_LOOP] = disp[BC_ILOOP];
  72   disp[BC_FUNCF] = disp[BC_IFUNCF];
  73   disp[BC_FUNCV] = disp[BC_IFUNCV];
  74   GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  75   for (i = 0; i < GG_NUM_ASMFF; i++)
  76     GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
  77 #if LJ_TARGET_MIPS
  78   memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
  79 #endif
  80 }
  81
  82 #if LJ_HASJIT
  83 /* Initialize hotcount table. */
  84 void lj_dispatch_init_hotcount(global_State *g)
  85 {
  86   int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  87   HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  88   HotCount *hotcount = G2GG(g)->hotcount;
  89   uint32_t i;
  90   for (i = 0; i < HOTCOUNT_SIZE; i++)
  91     hotcount[i] = start;
  92 }
  93 #endif
  94
  95 /* Internal dispatch mode bits. */
  96 #define DISPMODE_CALL   0x01    /* Override call dispatch. */
  97 #define DISPMODE_RET    0x02    /* Override return dispatch. */
  98 #define DISPMODE_INS    0x04    /* Override instruction dispatch. */
  99 #define DISPMODE_JIT    0x10    /* JIT compiler on. */
 100 #define DISPMODE_REC    0x20    /* Recording active. */
 101 #define DISPMODE_PROF   0x40    /* Profiling active. */
 102
 103 /* Update dispatch table depending on various flags. */
 104 void lj_dispatch_update(global_State *g)
 105 {
 106   uint8_t oldmode = g->dispatchmode;
 107   uint8_t mode = 0;
 108 #if LJ_HASJIT
 109   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
 110   mode |= G2J(g)->state != LJ_TRACE_IDLE ?
 111             (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
 112 #endif
 113 #if LJ_HASPROFILE
 114   mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
 115 #endif
 116   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
 117   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
 118   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
 119   if (oldmode != mode) {  /* Mode changed? */
 120     ASMFunction *disp = G2GG(g)->dispatch;
 121     ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
 122     g->dispatchmode = mode;
 123
 124     /* Hotcount if JIT is on, but not while recording. */
 125     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
 126       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
 127       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
 128       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
 129       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
 130       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
 131     } else {  /* Otherwise use the non-hotcounting instructions. */
 132       f_forl = disp[GG_LEN_DDISP+BC_IFORL];
 133       f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
 134       f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
 135       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
 136       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
 137     }
 138     /* Init static counting instruction dispatch first (may be copied below). */
 139     disp[GG_LEN_DDISP+BC_FORL] = f_forl;
 140     disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
 141     disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
 142
 143     /* Set dynamic instruction dispatch. */
 144     if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
 145       /* Need to update the whole table. */
 146       if (!(mode & DISPMODE_INS)) {  /* No ins dispatch? */
 147         /* Copy static dispatch table to dynamic dispatch table. */
 148         memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
 149         /* Overwrite with dynamic return dispatch. */
 150         if ((mode & DISPMODE_RET)) {
 151           disp[BC_RETM] = lj_vm_rethook;
 152           disp[BC_RET] = lj_vm_rethook;
 153           disp[BC_RET0] = lj_vm_rethook;
 154           disp[BC_RET1] = lj_vm_rethook;
 155         }
 156       } else {
 157         /* The recording dispatch also checks for hooks. */
 158         ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
 159                         (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
 160         uint32_t i;
 161         for (i = 0; i < GG_LEN_SDISP; i++)
 162           disp[i] = f;
 163       }
 164     } else if (!(mode & DISPMODE_INS)) {
 165       /* Otherwise set dynamic counting ins. */
 166       disp[BC_FORL] = f_forl;
 167       disp[BC_ITERL] = f_iterl;
 168       disp[BC_LOOP] = f_loop;
 169       /* Set dynamic return dispatch. */
 170       if ((mode & DISPMODE_RET)) {
 171         disp[BC_RETM] = lj_vm_rethook;
 172         disp[BC_RET] = lj_vm_rethook;
 173         disp[BC_RET0] = lj_vm_rethook;
 174         disp[BC_RET1] = lj_vm_rethook;
 175       } else {
 176         disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
 177         disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
 178         disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
 179         disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
 180       }
 181     }
 182
 183     /* Set dynamic call dispatch. */
 184     if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
 185       uint32_t i;
 186       if ((mode & DISPMODE_CALL) == 0) {  /* No call hooks? */
 187         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 188           disp[i] = makeasmfunc(lj_bc_ofs[i]);
 189       } else {
 190         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 191           disp[i] = lj_vm_callhook;
 192       }
 193     }
 194     if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
 195       disp[BC_FUNCF] = f_funcf;
 196       disp[BC_FUNCV] = f_funcv;
 197     }
 198
 199 #if LJ_HASJIT
 200     /* Reset hotcounts for JIT off to on transition. */
 201     if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
 202       lj_dispatch_init_hotcount(g);
 203 #endif
 204   }
 205 }
 206
 207 /* -- JIT mode setting ---------------------------------------------------- */
 208
 209 #if LJ_HASJIT
 210 /* Set JIT mode for a single prototype. */
 211 static void setptmode(global_State *g, GCproto *pt, int mode)
 212 {
 213   if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
 214     pt->flags &= ~PROTO_NOJIT;
 215     lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
 216   } else {  /* Flush and/or disable JIT compilation. */
 217     if (!(mode & LUAJIT_MODE_FLUSH))
 218       pt->flags |= PROTO_NOJIT;
 219     lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
 220   }
 221 }
 222
 223 /* Recursively set the JIT mode for all children of a prototype. */
 224 static void setptmode_all(global_State *g, GCproto *pt, int mode)
 225 {
 226   ptrdiff_t i;
 227   if (!(pt->flags & PROTO_CHILD)) return;
 228   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
 229     GCobj *o = proto_kgc(pt, i);
 230     if (o->gch.gct == ~LJ_TPROTO) {
 231       setptmode(g, gco2pt(o), mode);
 232       setptmode_all(g, gco2pt(o), mode);
 233     }
 234   }
 235 }
 236 #endif
 237
 238 /* Public API function: control the JIT engine. */
 239 int luaJIT_setmode(lua_State *L, int idx, int mode)
 240 {
 241   global_State *g = G(L);
 242   int mm = mode & LUAJIT_MODE_MASK;
 243   lj_trace_abort(g);  /* Abort recording on any state change. */
 244   /* Avoid pulling the rug from under our own feet. */
 245   if ((g->hookmask & HOOK_GC))
 246     lj_err_caller(L, LJ_ERR_NOGCMM);
 247   switch (mm) {
 248 #if LJ_HASJIT
 249   case LUAJIT_MODE_ENGINE:
 250     if ((mode & LUAJIT_MODE_FLUSH)) {
 251       lj_trace_flushall(L);
 252     } else {
 253       if (!(mode & LUAJIT_MODE_ON))
 254         G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
 255 #if LJ_TARGET_X86ORX64
 256       else if ((G2J(g)->flags & JIT_F_SSE2))
 257         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 258       else
 259         return 0;  /* Don't turn on JIT compiler without SSE2 support. */
 260 #else
 261       else
 262         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 263 #endif
 264       lj_dispatch_update(g);
 265     }
 266     break;
 267   case LUAJIT_MODE_FUNC:
 268   case LUAJIT_MODE_ALLFUNC:
 269   case LUAJIT_MODE_ALLSUBFUNC: {
 270     cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
 271                   idx > 0 ? L->base + (idx-1) : L->top + idx;
 272     GCproto *pt;
 273     if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
 274       pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
 275     else if (tvisproto(tv))
 276       pt = protoV(tv);
 277     else
 278       return 0;  /* Failed. */
 279     if (mm != LUAJIT_MODE_ALLSUBFUNC)
 280       setptmode(g, pt, mode);
 281     if (mm != LUAJIT_MODE_FUNC)
 282       setptmode_all(g, pt, mode);
 283     break;
 284     }
 285   case LUAJIT_MODE_TRACE:
 286     if (!(mode & LUAJIT_MODE_FLUSH))
 287       return 0;  /* Failed. */
 288     lj_trace_flush(G2J(g), idx);
 289     break;
 290 #else
 291   case LUAJIT_MODE_ENGINE:
 292   case LUAJIT_MODE_FUNC:
 293   case LUAJIT_MODE_ALLFUNC:
 294   case LUAJIT_MODE_ALLSUBFUNC:
 295     UNUSED(idx);
 296     if ((mode & LUAJIT_MODE_ON))
 297       return 0;  /* Failed. */
 298     break;
 299 #endif
 300   case LUAJIT_MODE_WRAPCFUNC:
 301     if ((mode & LUAJIT_MODE_ON)) {
 302       if (idx != 0) {
 303         cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
 304         if (tvislightud(tv))
 305           g->wrapf = (lua_CFunction)lightudV(tv);
 306         else
 307           return 0;  /* Failed. */
 308       } else {
 309         return 0;  /* Failed. */
 310       }
 311       g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
 312     } else {
 313       g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
 314     }
 315     break;
 316   default:
 317     return 0;  /* Failed. */
 318   }
 319   return 1;  /* OK. */
 320 }
 321
 322 /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
 323 LUA_API void LUAJIT_VERSION_SYM(void)
 324 {
 325 }
 326
 327 /* -- Hooks --------------------------------------------------------------- */
 328
 329 /* This function can be called asynchronously (e.g. during a signal). */
 330 LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
 331 {
 332   global_State *g = G(L);
 333   mask &= HOOK_EVENTMASK;
 334   if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
 335   g->hookf = func;
 336   g->hookcount = g->hookcstart = (int32_t)count;
 337   g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
 338   lj_trace_abort(g);  /* Abort recording on any hook change. */
 339   lj_dispatch_update(g);
 340   return 1;
 341 }
 342
 343 LUA_API lua_Hook lua_gethook(lua_State *L)
 344 {
 345   return G(L)->hookf;
 346 }
 347
 348 LUA_API int lua_gethookmask(lua_State *L)
 349 {
 350   return G(L)->hookmask & HOOK_EVENTMASK;
 351 }
 352
 353 LUA_API int lua_gethookcount(lua_State *L)
 354 {
 355   return (int)G(L)->hookcstart;
 356 }
 357
 358 /* Call a hook. */
 359 static void callhook(lua_State *L, int event, BCLine line)
 360 {
 361   global_State *g = G(L);
 362   lua_Hook hookf = g->hookf;
 363   if (hookf && !hook_active(g)) {
 364     lua_Debug ar;
 365     lj_trace_abort(g);  /* Abort recording on any hook call. */
 366     ar.event = event;
 367     ar.currentline = line;
 368     /* Top frame, nextframe = NULL. */
 369     ar.i_ci = (int)((L->base-1) - tvref(L->stack));
 370     lj_state_checkstack(L, 1+LUA_MINSTACK);
 371 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
 372     lj_profile_hook_enter(g);
 373 #else
 374     hook_enter(g);
 375 #endif
 376     hookf(L, &ar);
 377     lua_assert(hook_active(g));
 378     setgcref(g->cur_L, obj2gco(L));
 379 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
 380     lj_profile_hook_leave(g);
 381 #else
 382     hook_leave(g);
 383 #endif
 384   }
 385 }
 386
 387 /* -- Dispatch callbacks -------------------------------------------------- */
 388
 389 /* Calculate number of used stack slots in the current frame. */
 390 static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
 391 {
 392   BCIns ins = pc[-1];
 393   if (bc_op(ins) == BC_UCLO)
 394     ins = pc[bc_j(ins)];
 395   switch (bc_op(ins)) {
 396   case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
 397   case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
 398   case BC_TSETM: return bc_a(ins) + nres-1;
 399   default: return pt->framesize;
 400   }
 401 }
 402
 403 /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
 404 void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
 405 {
 406   ERRNO_SAVE
 407   GCfunc *fn = curr_func(L);
 408   GCproto *pt = funcproto(fn);
 409   void *cf = cframe_raw(L->cframe);
 410   const BCIns *oldpc = cframe_pc(cf);
 411   global_State *g = G(L);
 412   BCReg slots;
 413   setcframe_pc(cf, pc);
 414   slots = cur_topslot(pt, pc, cframe_multres_n(cf));
 415   L->top = L->base + slots;  /* Fix top. */
 416 #if LJ_HASJIT
 417   {
 418     jit_State *J = G2J(g);
 419     if (J->state != LJ_TRACE_IDLE) {
 420 #ifdef LUA_USE_ASSERT
 421       ptrdiff_t delta = L->top - L->base;
 422 #endif
 423       J->L = L;
 424       lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 425       lua_assert(L->top - L->base == delta);
 426     }
 427   }
 428 #endif
 429   if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
 430     g->hookcount = g->hookcstart;
 431     callhook(L, LUA_HOOKCOUNT, -1);
 432     L->top = L->base + slots;  /* Fix top again. */
 433   }
 434   if ((g->hookmask & LUA_MASKLINE)) {
 435     BCPos npc = proto_bcpos(pt, pc) - 1;
 436     BCPos opc = proto_bcpos(pt, oldpc) - 1;
 437     BCLine line = lj_debug_line(pt, npc);
 438     if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
 439       callhook(L, LUA_HOOKLINE, line);
 440       L->top = L->base + slots;  /* Fix top again. */
 441     }
 442   }
 443   if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
 444     callhook(L, LUA_HOOKRET, -1);
 445   ERRNO_RESTORE
 446 }
 447
 448 /* Initialize call. Ensure stack space and return # of missing parameters. */
 449 static int call_init(lua_State *L, GCfunc *fn)
 450 {
 451   if (isluafunc(fn)) {
 452     GCproto *pt = funcproto(fn);
 453     int numparams = pt->numparams;
 454     int gotparams = (int)(L->top - L->base);
 455     int need = pt->framesize;
 456     if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
 457     lj_state_checkstack(L, (MSize)need);
 458     numparams -= gotparams;
 459     return numparams >= 0 ? numparams : 0;
 460   } else {
 461     lj_state_checkstack(L, LUA_MINSTACK);
 462     return 0;
 463   }
 464 }
 465
 466 /* Call dispatch. Used by call hooks, hot calls or when recording. */
 467 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 468 {
 469   ERRNO_SAVE
 470   GCfunc *fn = curr_func(L);
 471   BCOp op;
 472   global_State *g = G(L);
 473 #if LJ_HASJIT
 474   jit_State *J = G2J(g);
 475 #endif
 476   int missing = call_init(L, fn);
 477 #if LJ_HASJIT
 478   J->L = L;
 479   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
 480 #ifdef LUA_USE_ASSERT
 481     ptrdiff_t delta = L->top - L->base;
 482 #endif
 483     pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
 484     lj_trace_hot(J, pc);
 485     lua_assert(L->top - L->base == delta);
 486     goto out;
 487   } else if (J->state != LJ_TRACE_IDLE &&
 488              !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 489 #ifdef LUA_USE_ASSERT
 490     ptrdiff_t delta = L->top - L->base;
 491 #endif
 492     /* Record the FUNC* bytecodes, too. */
 493     lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 494     lua_assert(L->top - L->base == delta);
 495   }
 496 #endif
 497   if ((g->hookmask & LUA_MASKCALL)) {
 498     int i;
 499     for (i = 0; i < missing; i++)  /* Add missing parameters. */
 500       setnilV(L->top++);
 501     callhook(L, LUA_HOOKCALL, -1);
 502     /* Preserve modifications of missing parameters by lua_setlocal(). */
 503     while (missing-- > 0 && tvisnil(L->top - 1))
 504       L->top--;
 505   }
 506 #if LJ_HASJIT
 507 out:
 508 #endif
 509   op = bc_op(pc[-1]);  /* Get FUNC* op. */
 510 #if LJ_HASJIT
 511   /* Use the non-hotcounting variants if JIT is off or while recording. */
 512   if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
 513       (op == BC_FUNCF || op == BC_FUNCV))
 514     op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
 515 #endif
 516   ERRNO_RESTORE
 517   return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
 518 }
 519
 520 #if LJ_HASJIT
 521 /* Stitch a new trace. */
 522 void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
 523 {
 524   ERRNO_SAVE
 525   lua_State *L = J->L;
 526   void *cf = cframe_raw(L->cframe);
 527   const BCIns *oldpc = cframe_pc(cf);
 528   setcframe_pc(cf, pc);
 529   /* Before dispatch, have to bias PC by 1. */
 530   L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
 531   lj_trace_stitch(J, pc-1);  /* Point to the CALL instruction. */
 532   setcframe_pc(cf, oldpc);
 533   ERRNO_RESTORE
 534 }
 535 #endif
 536
 537 #if LJ_HASPROFILE
 538 /* Profile dispatch. */
 539 void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
 540 {
 541   ERRNO_SAVE
 542   GCfunc *fn = curr_func(L);
 543   GCproto *pt = funcproto(fn);
 544   void *cf = cframe_raw(L->cframe);
 545   const BCIns *oldpc = cframe_pc(cf);
 546   global_State *g;
 547   setcframe_pc(cf, pc);
 548   L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
 549   lj_profile_interpreter(L);
 550   setcframe_pc(cf, oldpc);
 551   g = G(L);
 552   setgcref(g->cur_L, obj2gco(L));
 553   setvmstate(g, INTERP);
 554   ERRNO_RESTORE
 555 }
 556 #endif
 557