src/lj_dispatch.c

   1 /*
   2 ** Instruction dispatch handling.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_dispatch_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10 #include "lj_err.h"
  11 #include "lj_buf.h"
  12 #include "lj_func.h"
  13 #include "lj_str.h"
  14 #include "lj_tab.h"
  15 #include "lj_meta.h"
  16 #include "lj_debug.h"
  17 #include "lj_state.h"
  18 #include "lj_frame.h"
  19 #include "lj_bc.h"
  20 #include "lj_ff.h"
  21 #include "lj_strfmt.h"
  22 #if LJ_HASJIT
  23 #include "lj_jit.h"
  24 #endif
  25 #if LJ_HASFFI
  26 #include "lj_ccallback.h"
  27 #endif
  28 #include "lj_trace.h"
  29 #include "lj_dispatch.h"
  30 #if LJ_HASPROFILE
  31 #include "lj_profile.h"
  32 #endif
  33 #include "lj_vm.h"
  34 #include "luajit.h"
  35
  36 /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
  37 LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
  38
  39 /* -- Dispatch table management ------------------------------------------- */
  40
  41 #if LJ_TARGET_MIPS
  42 #include <math.h>
  43 LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
  44                                                           lua_State *co);
  45 #if !LJ_HASJIT
  46 #define lj_dispatch_stitch      lj_dispatch_ins
  47 #endif
  48 #if !LJ_HASPROFILE
  49 #define lj_dispatch_profile     lj_dispatch_ins
  50 #endif
  51
  52 #define GOTFUNC(name)   (ASMFunction)name,
  53 static const ASMFunction dispatch_got[] = {
  54   GOTDEF(GOTFUNC)
  55 };
  56 #undef GOTFUNC
  57 #endif
  58
  59 /* Initialize instruction dispatch table and hot counters. */
  60 void lj_dispatch_init(GG_State *GG)
  61 {
  62   uint32_t i;
  63   ASMFunction *disp = GG->dispatch;
  64   for (i = 0; i < GG_LEN_SDISP; i++)
  65     disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  66   for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  67     disp[i] = makeasmfunc(lj_bc_ofs[i]);
  68   /* The JIT engine is off by default. luaopen_jit() turns it on. */
  69   disp[BC_FORL] = disp[BC_IFORL];
  70   disp[BC_ITERL] = disp[BC_IITERL];
  71   /* Workaround for stable v2.1 bytecode. TODO: Replace with BC_IITERN. */
  72   disp[BC_ITERN] = &lj_vm_IITERN;
  73   disp[BC_LOOP] = disp[BC_ILOOP];
  74   disp[BC_FUNCF] = disp[BC_IFUNCF];
  75   disp[BC_FUNCV] = disp[BC_IFUNCV];
  76   GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  77   for (i = 0; i < GG_NUM_ASMFF; i++)
  78     GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
  79 #if LJ_TARGET_MIPS
  80   memcpy(GG->got, dispatch_got, LJ_GOT__MAX*sizeof(ASMFunction *));
  81 #endif
  82 }
  83
  84 #if LJ_HASJIT
  85 /* Initialize hotcount table. */
  86 void lj_dispatch_init_hotcount(global_State *g)
  87 {
  88   int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  89   HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  90   HotCount *hotcount = G2GG(g)->hotcount;
  91   uint32_t i;
  92   for (i = 0; i < HOTCOUNT_SIZE; i++)
  93     hotcount[i] = start;
  94 }
  95 #endif
  96
  97 /* Internal dispatch mode bits. */
  98 #define DISPMODE_CALL   0x01    /* Override call dispatch. */
  99 #define DISPMODE_RET    0x02    /* Override return dispatch. */
 100 #define DISPMODE_INS    0x04    /* Override instruction dispatch. */
 101 #define DISPMODE_JIT    0x10    /* JIT compiler on. */
 102 #define DISPMODE_REC    0x20    /* Recording active. */
 103 #define DISPMODE_PROF   0x40    /* Profiling active. */
 104
 105 /* Update dispatch table depending on various flags. */
 106 void lj_dispatch_update(global_State *g)
 107 {
 108   uint8_t oldmode = g->dispatchmode;
 109   uint8_t mode = 0;
 110 #if LJ_HASJIT
 111   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
 112   mode |= G2J(g)->state != LJ_TRACE_IDLE ?
 113             (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
 114 #endif
 115 #if LJ_HASPROFILE
 116   mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
 117 #endif
 118   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
 119   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
 120   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
 121   if (oldmode != mode) {  /* Mode changed? */
 122     ASMFunction *disp = G2GG(g)->dispatch;
 123     ASMFunction f_forl, f_iterl, f_itern, f_loop, f_funcf, f_funcv;
 124     g->dispatchmode = mode;
 125
 126     /* Hotcount if JIT is on, but not while recording. */
 127     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
 128       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
 129       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
 130       f_itern = makeasmfunc(lj_bc_ofs[BC_ITERN]);
 131       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
 132       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
 133       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
 134     } else {  /* Otherwise use the non-hotcounting instructions. */
 135       f_forl = disp[GG_LEN_DDISP+BC_IFORL];
 136       f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
 137       f_itern = &lj_vm_IITERN;
 138       f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
 139       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
 140       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
 141     }
 142     /* Init static counting instruction dispatch first (may be copied below). */
 143     disp[GG_LEN_DDISP+BC_FORL] = f_forl;
 144     disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
 145     disp[GG_LEN_DDISP+BC_ITERN] = f_itern;
 146     disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
 147
 148     /* Set dynamic instruction dispatch. */
 149     if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
 150       /* Need to update the whole table. */
 151       if (!(mode & DISPMODE_INS)) {  /* No ins dispatch? */
 152         /* Copy static dispatch table to dynamic dispatch table. */
 153         memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
 154         /* Overwrite with dynamic return dispatch. */
 155         if ((mode & DISPMODE_RET)) {
 156           disp[BC_RETM] = lj_vm_rethook;
 157           disp[BC_RET] = lj_vm_rethook;
 158           disp[BC_RET0] = lj_vm_rethook;
 159           disp[BC_RET1] = lj_vm_rethook;
 160         }
 161       } else {
 162         /* The recording dispatch also checks for hooks. */
 163         ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
 164                         (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
 165         uint32_t i;
 166         for (i = 0; i < GG_LEN_SDISP; i++)
 167           disp[i] = f;
 168       }
 169     } else if (!(mode & DISPMODE_INS)) {
 170       /* Otherwise set dynamic counting ins. */
 171       disp[BC_FORL] = f_forl;
 172       disp[BC_ITERL] = f_iterl;
 173       disp[BC_ITERN] = f_itern;
 174       disp[BC_LOOP] = f_loop;
 175       /* Set dynamic return dispatch. */
 176       if ((mode & DISPMODE_RET)) {
 177         disp[BC_RETM] = lj_vm_rethook;
 178         disp[BC_RET] = lj_vm_rethook;
 179         disp[BC_RET0] = lj_vm_rethook;
 180         disp[BC_RET1] = lj_vm_rethook;
 181       } else {
 182         disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
 183         disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
 184         disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
 185         disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
 186       }
 187     }
 188
 189     /* Set dynamic call dispatch. */
 190     if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
 191       uint32_t i;
 192       if ((mode & DISPMODE_CALL) == 0) {  /* No call hooks? */
 193         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 194           disp[i] = makeasmfunc(lj_bc_ofs[i]);
 195       } else {
 196         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 197           disp[i] = lj_vm_callhook;
 198       }
 199     }
 200     if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
 201       disp[BC_FUNCF] = f_funcf;
 202       disp[BC_FUNCV] = f_funcv;
 203     }
 204
 205 #if LJ_HASJIT
 206     /* Reset hotcounts for JIT off to on transition. */
 207     if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
 208       lj_dispatch_init_hotcount(g);
 209 #endif
 210   }
 211 }
 212
 213 /* -- JIT mode setting ---------------------------------------------------- */
 214
 215 #if LJ_HASJIT
 216 /* Set JIT mode for a single prototype. */
 217 static void setptmode(global_State *g, GCproto *pt, int mode)
 218 {
 219   if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
 220     pt->flags &= ~PROTO_NOJIT;
 221     lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
 222   } else {  /* Flush and/or disable JIT compilation. */
 223     if (!(mode & LUAJIT_MODE_FLUSH))
 224       pt->flags |= PROTO_NOJIT;
 225     lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
 226   }
 227 }
 228
 229 /* Recursively set the JIT mode for all children of a prototype. */
 230 static void setptmode_all(global_State *g, GCproto *pt, int mode)
 231 {
 232   ptrdiff_t i;
 233   if (!(pt->flags & PROTO_CHILD)) return;
 234   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
 235     GCobj *o = proto_kgc(pt, i);
 236     if (o->gch.gct == ~LJ_TPROTO) {
 237       setptmode(g, gco2pt(o), mode);
 238       setptmode_all(g, gco2pt(o), mode);
 239     }
 240   }
 241 }
 242 #endif
 243
 244 /* Public API function: control the JIT engine. */
 245 int luaJIT_setmode(lua_State *L, int idx, int mode)
 246 {
 247   global_State *g = G(L);
 248   int mm = mode & LUAJIT_MODE_MASK;
 249   lj_trace_abort(g);  /* Abort recording on any state change. */
 250   /* Avoid pulling the rug from under our own feet. */
 251   if ((g->hookmask & HOOK_GC))
 252     lj_err_caller(L, LJ_ERR_NOGCMM);
 253   switch (mm) {
 254 #if LJ_HASJIT
 255   case LUAJIT_MODE_ENGINE:
 256     if ((mode & LUAJIT_MODE_FLUSH)) {
 257       lj_trace_flushall(L);
 258     } else {
 259       if (!(mode & LUAJIT_MODE_ON))
 260         G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
 261       else
 262         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 263       lj_dispatch_update(g);
 264     }
 265     break;
 266   case LUAJIT_MODE_FUNC:
 267   case LUAJIT_MODE_ALLFUNC:
 268   case LUAJIT_MODE_ALLSUBFUNC: {
 269     cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :
 270                   idx > 0 ? L->base + (idx-1) : L->top + idx;
 271     GCproto *pt;
 272     if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
 273       pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
 274     else if (tvisproto(tv))
 275       pt = protoV(tv);
 276     else
 277       return 0;  /* Failed. */
 278     if (mm != LUAJIT_MODE_ALLSUBFUNC)
 279       setptmode(g, pt, mode);
 280     if (mm != LUAJIT_MODE_FUNC)
 281       setptmode_all(g, pt, mode);
 282     break;
 283     }
 284   case LUAJIT_MODE_TRACE:
 285     if (!(mode & LUAJIT_MODE_FLUSH))
 286       return 0;  /* Failed. */
 287     lj_trace_flush(G2J(g), idx);
 288     break;
 289 #else
 290   case LUAJIT_MODE_ENGINE:
 291   case LUAJIT_MODE_FUNC:
 292   case LUAJIT_MODE_ALLFUNC:
 293   case LUAJIT_MODE_ALLSUBFUNC:
 294     UNUSED(idx);
 295     if ((mode & LUAJIT_MODE_ON))
 296       return 0;  /* Failed. */
 297     break;
 298 #endif
 299   case LUAJIT_MODE_WRAPCFUNC:
 300     if ((mode & LUAJIT_MODE_ON)) {
 301       if (idx != 0) {
 302         cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
 303         if (tvislightud(tv))
 304           g->wrapf = (lua_CFunction)lightudV(g, tv);
 305         else
 306           return 0;  /* Failed. */
 307       } else {
 308         return 0;  /* Failed. */
 309       }
 310       setbc_op(&g->bc_cfunc_ext, BC_FUNCCW);
 311     } else {
 312       setbc_op(&g->bc_cfunc_ext, BC_FUNCC);
 313     }
 314     break;
 315   default:
 316     return 0;  /* Failed. */
 317   }
 318   return 1;  /* OK. */
 319 }
 320
 321 /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
 322 LUA_API void LUAJIT_VERSION_SYM(void)
 323 {
 324 }
 325
 326 /* -- Hooks --------------------------------------------------------------- */
 327
 328 /* This function can be called asynchronously (e.g. during a signal). */
 329 LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
 330 {
 331   global_State *g = G(L);
 332   mask &= HOOK_EVENTMASK;
 333   if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
 334   g->hookf = func;
 335   g->hookcount = g->hookcstart = (int32_t)count;
 336   g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
 337   lj_trace_abort(g);  /* Abort recording on any hook change. */
 338   lj_dispatch_update(g);
 339   return 1;
 340 }
 341
 342 LUA_API lua_Hook lua_gethook(lua_State *L)
 343 {
 344   return G(L)->hookf;
 345 }
 346
 347 LUA_API int lua_gethookmask(lua_State *L)
 348 {
 349   return G(L)->hookmask & HOOK_EVENTMASK;
 350 }
 351
 352 LUA_API int lua_gethookcount(lua_State *L)
 353 {
 354   return (int)G(L)->hookcstart;
 355 }
 356
 357 /* Call a hook. */
 358 static void callhook(lua_State *L, int event, BCLine line)
 359 {
 360   global_State *g = G(L);
 361   lua_Hook hookf = g->hookf;
 362   if (hookf && !hook_active(g)) {
 363     lua_Debug ar;
 364     lj_trace_abort(g);  /* Abort recording on any hook call. */
 365     ar.event = event;
 366     ar.currentline = line;
 367     /* Top frame, nextframe = NULL. */
 368     ar.i_ci = (int)((L->base-1) - tvref(L->stack));
 369     lj_state_checkstack(L, 1+LUA_MINSTACK);
 370 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
 371     lj_profile_hook_enter(g);
 372 #else
 373     hook_enter(g);
 374 #endif
 375     hookf(L, &ar);
 376     lj_assertG(hook_active(g), "active hook flag removed");
 377     setgcref(g->cur_L, obj2gco(L));
 378 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
 379     lj_profile_hook_leave(g);
 380 #else
 381     hook_leave(g);
 382 #endif
 383   }
 384 }
 385
 386 /* -- Dispatch callbacks -------------------------------------------------- */
 387
 388 /* Calculate number of used stack slots in the current frame. */
 389 static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
 390 {
 391   BCIns ins = pc[-1];
 392   if (bc_op(ins) == BC_UCLO)
 393     ins = pc[bc_j(ins)];
 394   switch (bc_op(ins)) {
 395   case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
 396   case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
 397   case BC_TSETM: return bc_a(ins) + nres-1;
 398   default: return pt->framesize;
 399   }
 400 }
 401
 402 /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
 403 void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
 404 {
 405   ERRNO_SAVE
 406   GCfunc *fn = curr_func(L);
 407   GCproto *pt = funcproto(fn);
 408   void *cf = cframe_raw(L->cframe);
 409   const BCIns *oldpc = cframe_pc(cf);
 410   global_State *g = G(L);
 411   BCReg slots;
 412   setcframe_pc(cf, pc);
 413   slots = cur_topslot(pt, pc, cframe_multres_n(cf));
 414   L->top = L->base + slots;  /* Fix top. */
 415 #if LJ_HASJIT
 416   {
 417     jit_State *J = G2J(g);
 418     if (J->state != LJ_TRACE_IDLE) {
 419 #ifdef LUA_USE_ASSERT
 420       ptrdiff_t delta = L->top - L->base;
 421 #endif
 422       J->L = L;
 423       lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 424       lj_assertG(L->top - L->base == delta,
 425                  "unbalanced stack after tracing of instruction");
 426     }
 427   }
 428 #endif
 429   if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
 430     g->hookcount = g->hookcstart;
 431     callhook(L, LUA_HOOKCOUNT, -1);
 432     L->top = L->base + slots;  /* Fix top again. */
 433   }
 434   if ((g->hookmask & LUA_MASKLINE)) {
 435     BCPos npc = proto_bcpos(pt, pc) - 1;
 436     BCPos opc = proto_bcpos(pt, oldpc) - 1;
 437     BCLine line = lj_debug_line(pt, npc);
 438     if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
 439       callhook(L, LUA_HOOKLINE, line);
 440       L->top = L->base + slots;  /* Fix top again. */
 441     }
 442   }
 443   if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
 444     callhook(L, LUA_HOOKRET, -1);
 445   ERRNO_RESTORE
 446 }
 447
 448 /* Initialize call. Ensure stack space and return # of missing parameters. */
 449 static int call_init(lua_State *L, GCfunc *fn)
 450 {
 451   if (isluafunc(fn)) {
 452     GCproto *pt = funcproto(fn);
 453     int numparams = pt->numparams;
 454     int gotparams = (int)(L->top - L->base);
 455     int need = pt->framesize;
 456     if ((pt->flags & PROTO_VARARG)) need += 1+LJ_FR2+gotparams;
 457     lj_state_checkstack(L, (MSize)need);
 458     numparams -= gotparams;
 459     return numparams >= 0 ? numparams : 0;
 460   } else {
 461     lj_state_checkstack(L, LUA_MINSTACK);
 462     return 0;
 463   }
 464 }
 465
 466 /* Call dispatch. Used by call hooks, hot calls or when recording. */
 467 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 468 {
 469   ERRNO_SAVE
 470   GCfunc *fn = curr_func(L);
 471   BCOp op;
 472   global_State *g = G(L);
 473 #if LJ_HASJIT
 474   jit_State *J = G2J(g);
 475 #endif
 476   int missing = call_init(L, fn);
 477 #if LJ_HASJIT
 478   J->L = L;
 479   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
 480 #ifdef LUA_USE_ASSERT
 481     ptrdiff_t delta = L->top - L->base;
 482 #endif
 483     pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
 484     lj_trace_hot(J, pc);
 485     lj_assertG(L->top - L->base == delta,
 486                "unbalanced stack after hot call");
 487     goto out;
 488   } else if (J->state != LJ_TRACE_IDLE &&
 489              !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 490 #ifdef LUA_USE_ASSERT
 491     ptrdiff_t delta = L->top - L->base;
 492 #endif
 493     /* Record the FUNC* bytecodes, too. */
 494     lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 495     lj_assertG(L->top - L->base == delta,
 496                "unbalanced stack after hot instruction");
 497   }
 498 #endif
 499   if ((g->hookmask & LUA_MASKCALL)) {
 500     int i;
 501     for (i = 0; i < missing; i++)  /* Add missing parameters. */
 502       setnilV(L->top++);
 503     callhook(L, LUA_HOOKCALL, -1);
 504     /* Preserve modifications of missing parameters by lua_setlocal(). */
 505     while (missing-- > 0 && tvisnil(L->top - 1))
 506       L->top--;
 507   }
 508 #if LJ_HASJIT
 509 out:
 510 #endif
 511   op = bc_op(pc[-1]);  /* Get FUNC* op. */
 512 #if LJ_HASJIT
 513   /* Use the non-hotcounting variants if JIT is off or while recording. */
 514   if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
 515       (op == BC_FUNCF || op == BC_FUNCV))
 516     op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
 517 #endif
 518   ERRNO_RESTORE
 519   return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
 520 }
 521
 522 #if LJ_HASJIT
 523 /* Stitch a new trace. */
 524 void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
 525 {
 526   ERRNO_SAVE
 527   lua_State *L = J->L;
 528   void *cf = cframe_raw(L->cframe);
 529   const BCIns *oldpc = cframe_pc(cf);
 530   setcframe_pc(cf, pc);
 531   /* Before dispatch, have to bias PC by 1. */
 532   L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
 533   lj_trace_stitch(J, pc-1);  /* Point to the CALL instruction. */
 534   setcframe_pc(cf, oldpc);
 535   ERRNO_RESTORE
 536 }
 537 #endif
 538
 539 #if LJ_HASPROFILE
 540 /* Profile dispatch. */
 541 void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
 542 {
 543   ERRNO_SAVE
 544   GCfunc *fn = curr_func(L);
 545   GCproto *pt = funcproto(fn);
 546   void *cf = cframe_raw(L->cframe);
 547   const BCIns *oldpc = cframe_pc(cf);
 548   global_State *g;
 549   setcframe_pc(cf, pc);
 550   L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
 551   lj_profile_interpreter(L);
 552   setcframe_pc(cf, oldpc);
 553   g = G(L);
 554   setgcref(g->cur_L, obj2gco(L));
 555   setvmstate(g, INTERP);
 556   ERRNO_RESTORE
 557 }
 558 #endif
 559