src/lj_dispatch.c

   1 /*
   2 ** Instruction dispatch handling.
   3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_dispatch_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10 #include "lj_err.h"
  11 #include "lj_debug.h"
  12 #include "lj_state.h"
  13 #include "lj_frame.h"
  14 #include "lj_bc.h"
  15 #include "lj_ff.h"
  16 #if LJ_HASJIT
  17 #include "lj_jit.h"
  18 #endif
  19 #include "lj_trace.h"
  20 #include "lj_dispatch.h"
  21 #include "lj_vm.h"
  22 #include "luajit.h"
  23
  24 /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
  25 LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
  26
  27 /* -- Dispatch table management ------------------------------------------- */
  28
  29 /* Initialize instruction dispatch table and hot counters. */
  30 void lj_dispatch_init(GG_State *GG)
  31 {
  32   uint32_t i;
  33   ASMFunction *disp = GG->dispatch;
  34   for (i = 0; i < GG_LEN_SDISP; i++)
  35     disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  36   for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  37     disp[i] = makeasmfunc(lj_bc_ofs[i]);
  38   /* The JIT engine is off by default. luaopen_jit() turns it on. */
  39   disp[BC_FORL] = disp[BC_IFORL];
  40   disp[BC_ITERL] = disp[BC_IITERL];
  41   disp[BC_LOOP] = disp[BC_ILOOP];
  42   disp[BC_FUNCF] = disp[BC_IFUNCF];
  43   disp[BC_FUNCV] = disp[BC_IFUNCV];
  44   GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  45   for (i = 0; i < GG_NUM_ASMFF; i++)
  46     GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
  47 }
  48
  49 #if LJ_HASJIT
  50 /* Initialize hotcount table. */
  51 void lj_dispatch_init_hotcount(global_State *g)
  52 {
  53   int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  54   HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  55   HotCount *hotcount = G2GG(g)->hotcount;
  56   uint32_t i;
  57   for (i = 0; i < HOTCOUNT_SIZE; i++)
  58     hotcount[i] = start;
  59 }
  60 #endif
  61
  62 /* Internal dispatch mode bits. */
  63 #define DISPMODE_JIT    0x01    /* JIT compiler on. */
  64 #define DISPMODE_REC    0x02    /* Recording active. */
  65 #define DISPMODE_INS    0x04    /* Override instruction dispatch. */
  66 #define DISPMODE_CALL   0x08    /* Override call dispatch. */
  67 #define DISPMODE_RET    0x08    /* Override return dispatch. */
  68
  69 /* Update dispatch table depending on various flags. */
  70 void lj_dispatch_update(global_State *g)
  71 {
  72   uint8_t oldmode = g->dispatchmode;
  73   uint8_t mode = 0;
  74 #if LJ_HASJIT
  75   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
  76   mode |= G2J(g)->state != LJ_TRACE_IDLE ?
  77             (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
  78 #endif
  79   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
  80   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
  81   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
  82   if (oldmode != mode) {  /* Mode changed? */
  83     ASMFunction *disp = G2GG(g)->dispatch;
  84     ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
  85     g->dispatchmode = mode;
  86
  87     /* Hotcount if JIT is on, but not while recording. */
  88     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
  89       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
  90       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
  91       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
  92       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
  93       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
  94     } else {  /* Otherwise use the non-hotcounting instructions. */
  95       f_forl = disp[GG_LEN_DDISP+BC_IFORL];
  96       f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
  97       f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
  98       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
  99       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
 100     }
 101     /* Init static counting instruction dispatch first (may be copied below). */
 102     disp[GG_LEN_DDISP+BC_FORL] = f_forl;
 103     disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
 104     disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
 105
 106     /* Set dynamic instruction dispatch. */
 107     if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) {
 108       /* Need to update the whole table. */
 109       if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {  /* No ins dispatch? */
 110         /* Copy static dispatch table to dynamic dispatch table. */
 111         memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
 112         /* Overwrite with dynamic return dispatch. */
 113         if ((mode & DISPMODE_RET)) {
 114           disp[BC_RETM] = lj_vm_rethook;
 115           disp[BC_RET] = lj_vm_rethook;
 116           disp[BC_RET0] = lj_vm_rethook;
 117           disp[BC_RET1] = lj_vm_rethook;
 118         }
 119       } else {
 120         /* The recording dispatch also checks for hooks. */
 121         ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
 122         uint32_t i;
 123         for (i = 0; i < GG_LEN_SDISP; i++)
 124           disp[i] = f;
 125       }
 126     } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {
 127       /* Otherwise set dynamic counting ins. */
 128       disp[BC_FORL] = f_forl;
 129       disp[BC_ITERL] = f_iterl;
 130       disp[BC_LOOP] = f_loop;
 131       /* Set dynamic return dispatch. */
 132       if ((mode & DISPMODE_RET)) {
 133         disp[BC_RETM] = lj_vm_rethook;
 134         disp[BC_RET] = lj_vm_rethook;
 135         disp[BC_RET0] = lj_vm_rethook;
 136         disp[BC_RET1] = lj_vm_rethook;
 137       } else {
 138         disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
 139         disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
 140         disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
 141         disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
 142       }
 143     }
 144
 145     /* Set dynamic call dispatch. */
 146     if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
 147       uint32_t i;
 148       if ((mode & 8) == 0) {  /* No call hooks? */
 149         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 150           disp[i] = makeasmfunc(lj_bc_ofs[i]);
 151       } else {
 152         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 153           disp[i] = lj_vm_callhook;
 154       }
 155     }
 156     if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
 157       disp[BC_FUNCF] = f_funcf;
 158       disp[BC_FUNCV] = f_funcv;
 159     }
 160
 161 #if LJ_HASJIT
 162     /* Reset hotcounts for JIT off to on transition. */
 163     if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
 164       lj_dispatch_init_hotcount(g);
 165 #endif
 166   }
 167 }
 168
 169 /* -- JIT mode setting ---------------------------------------------------- */
 170
 171 #if LJ_HASJIT
 172 /* Set JIT mode for a single prototype. */
 173 static void setptmode(global_State *g, GCproto *pt, int mode)
 174 {
 175   if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
 176     pt->flags &= ~PROTO_NOJIT;
 177     lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
 178   } else {  /* Flush and/or disable JIT compilation. */
 179     if (!(mode & LUAJIT_MODE_FLUSH))
 180       pt->flags |= PROTO_NOJIT;
 181     lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
 182   }
 183 }
 184
 185 /* Recursively set the JIT mode for all children of a prototype. */
 186 static void setptmode_all(global_State *g, GCproto *pt, int mode)
 187 {
 188   ptrdiff_t i;
 189   if (!(pt->flags & PROTO_CHILD)) return;
 190   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
 191     GCobj *o = proto_kgc(pt, i);
 192     if (o->gch.gct == ~LJ_TPROTO) {
 193       setptmode(g, gco2pt(o), mode);
 194       setptmode_all(g, gco2pt(o), mode);
 195     }
 196   }
 197 }
 198 #endif
 199
 200 /* Public API function: control the JIT engine. */
 201 int luaJIT_setmode(lua_State *L, int idx, int mode)
 202 {
 203   global_State *g = G(L);
 204   int mm = mode & LUAJIT_MODE_MASK;
 205   lj_trace_abort(g);  /* Abort recording on any state change. */
 206   /* Avoid pulling the rug from under our own feet. */
 207   if ((g->hookmask & HOOK_GC))
 208     lj_err_caller(L, LJ_ERR_NOGCMM);
 209   switch (mm) {
 210 #if LJ_HASJIT
 211   case LUAJIT_MODE_ENGINE:
 212     if ((mode & LUAJIT_MODE_FLUSH)) {
 213       lj_trace_flushall(L);
 214     } else {
 215       if (!(mode & LUAJIT_MODE_ON))
 216         G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
 217 #if LJ_TARGET_X86ORX64
 218       else if ((G2J(g)->flags & JIT_F_SSE2))
 219         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 220       else
 221         return 0;  /* Don't turn on JIT compiler without SSE2 support. */
 222 #else
 223       else
 224         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 225 #endif
 226       lj_dispatch_update(g);
 227     }
 228     break;
 229   case LUAJIT_MODE_FUNC:
 230   case LUAJIT_MODE_ALLFUNC:
 231   case LUAJIT_MODE_ALLSUBFUNC: {
 232     cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
 233                   idx > 0 ? L->base + (idx-1) : L->top + idx;
 234     GCproto *pt;
 235     if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
 236       pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
 237     else if (tvisproto(tv))
 238       pt = protoV(tv);
 239     else
 240       return 0;  /* Failed. */
 241     if (mm != LUAJIT_MODE_ALLSUBFUNC)
 242       setptmode(g, pt, mode);
 243     if (mm != LUAJIT_MODE_FUNC)
 244       setptmode_all(g, pt, mode);
 245     break;
 246     }
 247   case LUAJIT_MODE_TRACE:
 248     if (!(mode & LUAJIT_MODE_FLUSH))
 249       return 0;  /* Failed. */
 250     lj_trace_flush(G2J(g), idx);
 251     break;
 252 #else
 253   case LUAJIT_MODE_ENGINE:
 254   case LUAJIT_MODE_FUNC:
 255   case LUAJIT_MODE_ALLFUNC:
 256   case LUAJIT_MODE_ALLSUBFUNC:
 257     UNUSED(idx);
 258     if ((mode & LUAJIT_MODE_ON))
 259       return 0;  /* Failed. */
 260     break;
 261 #endif
 262   case LUAJIT_MODE_WRAPCFUNC:
 263     if ((mode & LUAJIT_MODE_ON)) {
 264       if (idx != 0) {
 265         cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
 266         if (tvislightud(tv))
 267           g->wrapf = (lua_CFunction)lightudV(tv);
 268         else
 269           return 0;  /* Failed. */
 270       } else {
 271         return 0;  /* Failed. */
 272       }
 273       g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
 274     } else {
 275       g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
 276     }
 277     break;
 278   default:
 279     return 0;  /* Failed. */
 280   }
 281   return 1;  /* OK. */
 282 }
 283
 284 /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
 285 LUA_API void LUAJIT_VERSION_SYM(void)
 286 {
 287 }
 288
 289 /* -- Hooks --------------------------------------------------------------- */
 290
 291 /* This function can be called asynchronously (e.g. during a signal). */
 292 LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
 293 {
 294   global_State *g = G(L);
 295   mask &= HOOK_EVENTMASK;
 296   if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
 297   g->hookf = func;
 298   g->hookcount = g->hookcstart = (int32_t)count;
 299   g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
 300   lj_trace_abort(g);  /* Abort recording on any hook change. */
 301   lj_dispatch_update(g);
 302   return 1;
 303 }
 304
 305 LUA_API lua_Hook lua_gethook(lua_State *L)
 306 {
 307   return G(L)->hookf;
 308 }
 309
 310 LUA_API int lua_gethookmask(lua_State *L)
 311 {
 312   return G(L)->hookmask & HOOK_EVENTMASK;
 313 }
 314
 315 LUA_API int lua_gethookcount(lua_State *L)
 316 {
 317   return (int)G(L)->hookcstart;
 318 }
 319
 320 /* Call a hook. */
 321 static void callhook(lua_State *L, int event, BCLine line)
 322 {
 323   global_State *g = G(L);
 324   lua_Hook hookf = g->hookf;
 325   if (hookf && !hook_active(g)) {
 326     lua_Debug ar;
 327     lj_trace_abort(g);  /* Abort recording on any hook call. */
 328     ar.event = event;
 329     ar.currentline = line;
 330     /* Top frame, nextframe = NULL. */
 331     ar.i_ci = (int)((L->base-1) - tvref(L->stack));
 332     lj_state_checkstack(L, 1+LUA_MINSTACK);
 333     hook_enter(g);
 334     hookf(L, &ar);
 335     lua_assert(hook_active(g));
 336     hook_leave(g);
 337   }
 338 }
 339
 340 /* -- Dispatch callbacks -------------------------------------------------- */
 341
 342 /* Calculate number of used stack slots in the current frame. */
 343 static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
 344 {
 345   BCIns ins = pc[-1];
 346   if (bc_op(ins) == BC_UCLO)
 347     ins = pc[bc_j(ins)];
 348   switch (bc_op(ins)) {
 349   case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1;
 350   case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
 351   case BC_TSETM: return bc_a(ins) + nres-1;
 352   default: return pt->framesize;
 353   }
 354 }
 355
 356 /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
 357 void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
 358 {
 359   ERRNO_SAVE
 360   GCfunc *fn = curr_func(L);
 361   GCproto *pt = funcproto(fn);
 362   void *cf = cframe_raw(L->cframe);
 363   const BCIns *oldpc = cframe_pc(cf);
 364   global_State *g = G(L);
 365   BCReg slots;
 366   setcframe_pc(cf, pc);
 367   slots = cur_topslot(pt, pc, cframe_multres_n(cf));
 368   L->top = L->base + slots;  /* Fix top. */
 369 #if LJ_HASJIT
 370   {
 371     jit_State *J = G2J(g);
 372     if (J->state != LJ_TRACE_IDLE) {
 373       J->L = L;
 374       lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 375     }
 376   }
 377 #endif
 378   if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
 379     g->hookcount = g->hookcstart;
 380     callhook(L, LUA_HOOKCOUNT, -1);
 381     L->top = L->base + slots;  /* Fix top again. */
 382   }
 383   if ((g->hookmask & LUA_MASKLINE)) {
 384     BCPos npc = proto_bcpos(pt, pc) - 1;
 385     BCPos opc = proto_bcpos(pt, oldpc) - 1;
 386     BCLine line = lj_debug_line(pt, npc);
 387     if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
 388       callhook(L, LUA_HOOKLINE, line);
 389       L->top = L->base + slots;  /* Fix top again. */
 390     }
 391   }
 392   if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
 393     callhook(L, LUA_HOOKRET, -1);
 394   ERRNO_RESTORE
 395 }
 396
 397 /* Initialize call. Ensure stack space and return # of missing parameters. */
 398 static int call_init(lua_State *L, GCfunc *fn)
 399 {
 400   if (isluafunc(fn)) {
 401     GCproto *pt = funcproto(fn);
 402     int numparams = pt->numparams;
 403     int gotparams = (int)(L->top - L->base);
 404     int need = pt->framesize;
 405     if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
 406     lj_state_checkstack(L, (MSize)need);
 407     numparams -= gotparams;
 408     return numparams >= 0 ? numparams : 0;
 409   } else {
 410     lj_state_checkstack(L, LUA_MINSTACK);
 411     return 0;
 412   }
 413 }
 414
 415 /* Call dispatch. Used by call hooks, hot calls or when recording. */
 416 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 417 {
 418   ERRNO_SAVE
 419   GCfunc *fn = curr_func(L);
 420   BCOp op;
 421   global_State *g = G(L);
 422 #if LJ_HASJIT
 423   jit_State *J = G2J(g);
 424 #endif
 425   int missing = call_init(L, fn);
 426 #if LJ_HASJIT
 427   J->L = L;
 428   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
 429     pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
 430     lj_trace_hot(J, pc);
 431     goto out;
 432   } else if (J->state != LJ_TRACE_IDLE &&
 433              !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 434     /* Record the FUNC* bytecodes, too. */
 435     lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 436   }
 437 #endif
 438   if ((g->hookmask & LUA_MASKCALL)) {
 439     int i;
 440     for (i = 0; i < missing; i++)  /* Add missing parameters. */
 441       setnilV(L->top++);
 442     callhook(L, LUA_HOOKCALL, -1);
 443     /* Preserve modifications of missing parameters by lua_setlocal(). */
 444     while (missing-- > 0 && tvisnil(L->top - 1))
 445       L->top--;
 446   }
 447 #if LJ_HASJIT
 448 out:
 449 #endif
 450   op = bc_op(pc[-1]);  /* Get FUNC* op. */
 451 #if LJ_HASJIT
 452   /* Use the non-hotcounting variants if JIT is off or while recording. */
 453   if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
 454       (op == BC_FUNCF || op == BC_FUNCV))
 455     op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
 456 #endif
 457   ERRNO_RESTORE
 458   return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
 459 }
 460