src/lj_dispatch.c

   1 /*
   2 ** Instruction dispatch handling.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_dispatch_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10 #include "lj_err.h"
  11 #include "lj_func.h"
  12 #include "lj_str.h"
  13 #include "lj_tab.h"
  14 #include "lj_meta.h"
  15 #include "lj_debug.h"
  16 #include "lj_state.h"
  17 #include "lj_frame.h"
  18 #include "lj_bc.h"
  19 #include "lj_ff.h"
  20 #if LJ_HASJIT
  21 #include "lj_jit.h"
  22 #endif
  23 #if LJ_HASFFI
  24 #include "lj_ccallback.h"
  25 #endif
  26 #include "lj_trace.h"
  27 #include "lj_dispatch.h"
  28 #include "lj_vm.h"
  29 #include "luajit.h"
  30
  31 /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
  32 LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
  33
  34 /* -- Dispatch table management ------------------------------------------- */
  35
  36 #if LJ_TARGET_MIPS
  37 #include <math.h>
  38 LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
  39                                                           lua_State *co);
  40
  41 #define GOTFUNC(name)   (ASMFunction)name,
  42 static const ASMFunction dispatch_got[] = {
  43   GOTDEF(GOTFUNC)
  44 };
  45 #undef GOTFUNC
  46 #endif
  47
  48 /* Initialize instruction dispatch table and hot counters. */
  49 void lj_dispatch_init(GG_State *GG)
  50 {
  51   uint32_t i;
  52   ASMFunction *disp = GG->dispatch;
  53   for (i = 0; i < GG_LEN_SDISP; i++)
  54     disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  55   for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  56     disp[i] = makeasmfunc(lj_bc_ofs[i]);
  57   /* The JIT engine is off by default. luaopen_jit() turns it on. */
  58   disp[BC_FORL] = disp[BC_IFORL];
  59   disp[BC_ITERL] = disp[BC_IITERL];
  60   disp[BC_LOOP] = disp[BC_ILOOP];
  61   disp[BC_FUNCF] = disp[BC_IFUNCF];
  62   disp[BC_FUNCV] = disp[BC_IFUNCV];
  63   GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  64   for (i = 0; i < GG_NUM_ASMFF; i++)
  65     GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
  66 #if LJ_TARGET_MIPS
  67   memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
  68 #endif
  69 }
  70
  71 #if LJ_HASJIT
  72 /* Initialize hotcount table. */
  73 void lj_dispatch_init_hotcount(global_State *g)
  74 {
  75   int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  76   HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  77   HotCount *hotcount = G2GG(g)->hotcount;
  78   uint32_t i;
  79   for (i = 0; i < HOTCOUNT_SIZE; i++)
  80     hotcount[i] = start;
  81 }
  82 #endif
  83
  84 /* Internal dispatch mode bits. */
  85 #define DISPMODE_JIT    0x01    /* JIT compiler on. */
  86 #define DISPMODE_REC    0x02    /* Recording active. */
  87 #define DISPMODE_INS    0x04    /* Override instruction dispatch. */
  88 #define DISPMODE_CALL   0x08    /* Override call dispatch. */
  89 #define DISPMODE_RET    0x10    /* Override return dispatch. */
  90
  91 /* Update dispatch table depending on various flags. */
  92 void lj_dispatch_update(global_State *g)
  93 {
  94   uint8_t oldmode = g->dispatchmode;
  95   uint8_t mode = 0;
  96 #if LJ_HASJIT
  97   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
  98   mode |= G2J(g)->state != LJ_TRACE_IDLE ?
  99             (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
 100 #endif
 101   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
 102   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
 103   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
 104   if (oldmode != mode) {  /* Mode changed? */
 105     ASMFunction *disp = G2GG(g)->dispatch;
 106     ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
 107     g->dispatchmode = mode;
 108
 109     /* Hotcount if JIT is on, but not while recording. */
 110     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
 111       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
 112       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
 113       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
 114       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
 115       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
 116     } else {  /* Otherwise use the non-hotcounting instructions. */
 117       f_forl = disp[GG_LEN_DDISP+BC_IFORL];
 118       f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
 119       f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
 120       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
 121       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
 122     }
 123     /* Init static counting instruction dispatch first (may be copied below). */
 124     disp[GG_LEN_DDISP+BC_FORL] = f_forl;
 125     disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
 126     disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
 127
 128     /* Set dynamic instruction dispatch. */
 129     if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) {
 130       /* Need to update the whole table. */
 131       if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {  /* No ins dispatch? */
 132         /* Copy static dispatch table to dynamic dispatch table. */
 133         memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
 134         /* Overwrite with dynamic return dispatch. */
 135         if ((mode & DISPMODE_RET)) {
 136           disp[BC_RETM] = lj_vm_rethook;
 137           disp[BC_RET] = lj_vm_rethook;
 138           disp[BC_RET0] = lj_vm_rethook;
 139           disp[BC_RET1] = lj_vm_rethook;
 140         }
 141       } else {
 142         /* The recording dispatch also checks for hooks. */
 143         ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
 144         uint32_t i;
 145         for (i = 0; i < GG_LEN_SDISP; i++)
 146           disp[i] = f;
 147       }
 148     } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {
 149       /* Otherwise set dynamic counting ins. */
 150       disp[BC_FORL] = f_forl;
 151       disp[BC_ITERL] = f_iterl;
 152       disp[BC_LOOP] = f_loop;
 153       /* Set dynamic return dispatch. */
 154       if ((mode & DISPMODE_RET)) {
 155         disp[BC_RETM] = lj_vm_rethook;
 156         disp[BC_RET] = lj_vm_rethook;
 157         disp[BC_RET0] = lj_vm_rethook;
 158         disp[BC_RET1] = lj_vm_rethook;
 159       } else {
 160         disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
 161         disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
 162         disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
 163         disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
 164       }
 165     }
 166
 167     /* Set dynamic call dispatch. */
 168     if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
 169       uint32_t i;
 170       if ((mode & DISPMODE_CALL) == 0) {  /* No call hooks? */
 171         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 172           disp[i] = makeasmfunc(lj_bc_ofs[i]);
 173       } else {
 174         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
 175           disp[i] = lj_vm_callhook;
 176       }
 177     }
 178     if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
 179       disp[BC_FUNCF] = f_funcf;
 180       disp[BC_FUNCV] = f_funcv;
 181     }
 182
 183 #if LJ_HASJIT
 184     /* Reset hotcounts for JIT off to on transition. */
 185     if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
 186       lj_dispatch_init_hotcount(g);
 187 #endif
 188   }
 189 }
 190
 191 /* -- JIT mode setting ---------------------------------------------------- */
 192
 193 #if LJ_HASJIT
 194 /* Set JIT mode for a single prototype. */
 195 static void setptmode(global_State *g, GCproto *pt, int mode)
 196 {
 197   if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
 198     pt->flags &= ~PROTO_NOJIT;
 199     lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
 200   } else {  /* Flush and/or disable JIT compilation. */
 201     if (!(mode & LUAJIT_MODE_FLUSH))
 202       pt->flags |= PROTO_NOJIT;
 203     lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
 204   }
 205 }
 206
 207 /* Recursively set the JIT mode for all children of a prototype. */
 208 static void setptmode_all(global_State *g, GCproto *pt, int mode)
 209 {
 210   ptrdiff_t i;
 211   if (!(pt->flags & PROTO_CHILD)) return;
 212   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
 213     GCobj *o = proto_kgc(pt, i);
 214     if (o->gch.gct == ~LJ_TPROTO) {
 215       setptmode(g, gco2pt(o), mode);
 216       setptmode_all(g, gco2pt(o), mode);
 217     }
 218   }
 219 }
 220 #endif
 221
 222 /* Public API function: control the JIT engine. */
 223 int luaJIT_setmode(lua_State *L, int idx, int mode)
 224 {
 225   global_State *g = G(L);
 226   int mm = mode & LUAJIT_MODE_MASK;
 227   lj_trace_abort(g);  /* Abort recording on any state change. */
 228   /* Avoid pulling the rug from under our own feet. */
 229   if ((g->hookmask & HOOK_GC))
 230     lj_err_caller(L, LJ_ERR_NOGCMM);
 231   switch (mm) {
 232 #if LJ_HASJIT
 233   case LUAJIT_MODE_ENGINE:
 234     if ((mode & LUAJIT_MODE_FLUSH)) {
 235       lj_trace_flushall(L);
 236     } else {
 237       if (!(mode & LUAJIT_MODE_ON))
 238         G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
 239 #if LJ_TARGET_X86ORX64
 240       else if ((G2J(g)->flags & JIT_F_SSE2))
 241         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 242       else
 243         return 0;  /* Don't turn on JIT compiler without SSE2 support. */
 244 #else
 245       else
 246         G2J(g)->flags |= (uint32_t)JIT_F_ON;
 247 #endif
 248       lj_dispatch_update(g);
 249     }
 250     break;
 251   case LUAJIT_MODE_FUNC:
 252   case LUAJIT_MODE_ALLFUNC:
 253   case LUAJIT_MODE_ALLSUBFUNC: {
 254     cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
 255                   idx > 0 ? L->base + (idx-1) : L->top + idx;
 256     GCproto *pt;
 257     if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
 258       pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
 259     else if (tvisproto(tv))
 260       pt = protoV(tv);
 261     else
 262       return 0;  /* Failed. */
 263     if (mm != LUAJIT_MODE_ALLSUBFUNC)
 264       setptmode(g, pt, mode);
 265     if (mm != LUAJIT_MODE_FUNC)
 266       setptmode_all(g, pt, mode);
 267     break;
 268     }
 269   case LUAJIT_MODE_TRACE:
 270     if (!(mode & LUAJIT_MODE_FLUSH))
 271       return 0;  /* Failed. */
 272     lj_trace_flush(G2J(g), idx);
 273     break;
 274 #else
 275   case LUAJIT_MODE_ENGINE:
 276   case LUAJIT_MODE_FUNC:
 277   case LUAJIT_MODE_ALLFUNC:
 278   case LUAJIT_MODE_ALLSUBFUNC:
 279     UNUSED(idx);
 280     if ((mode & LUAJIT_MODE_ON))
 281       return 0;  /* Failed. */
 282     break;
 283 #endif
 284   case LUAJIT_MODE_WRAPCFUNC:
 285     if ((mode & LUAJIT_MODE_ON)) {
 286       if (idx != 0) {
 287         cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
 288         if (tvislightud(tv))
 289           g->wrapf = (lua_CFunction)lightudV(tv);
 290         else
 291           return 0;  /* Failed. */
 292       } else {
 293         return 0;  /* Failed. */
 294       }
 295       setbc_op(&g->bc_cfunc_ext, BC_FUNCCW);
 296     } else {
 297       setbc_op(&g->bc_cfunc_ext, BC_FUNCC);
 298     }
 299     break;
 300   default:
 301     return 0;  /* Failed. */
 302   }
 303   return 1;  /* OK. */
 304 }
 305
 306 /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
 307 LUA_API void LUAJIT_VERSION_SYM(void)
 308 {
 309 }
 310
 311 /* -- Hooks --------------------------------------------------------------- */
 312
 313 /* This function can be called asynchronously (e.g. during a signal). */
 314 LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
 315 {
 316   global_State *g = G(L);
 317   mask &= HOOK_EVENTMASK;
 318   if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
 319   g->hookf = func;
 320   g->hookcount = g->hookcstart = (int32_t)count;
 321   g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
 322   lj_trace_abort(g);  /* Abort recording on any hook change. */
 323   lj_dispatch_update(g);
 324   return 1;
 325 }
 326
 327 LUA_API lua_Hook lua_gethook(lua_State *L)
 328 {
 329   return G(L)->hookf;
 330 }
 331
 332 LUA_API int lua_gethookmask(lua_State *L)
 333 {
 334   return G(L)->hookmask & HOOK_EVENTMASK;
 335 }
 336
 337 LUA_API int lua_gethookcount(lua_State *L)
 338 {
 339   return (int)G(L)->hookcstart;
 340 }
 341
 342 /* Call a hook. */
 343 static void callhook(lua_State *L, int event, BCLine line)
 344 {
 345   global_State *g = G(L);
 346   lua_Hook hookf = g->hookf;
 347   if (hookf && !hook_active(g)) {
 348     lua_Debug ar;
 349     lj_trace_abort(g);  /* Abort recording on any hook call. */
 350     ar.event = event;
 351     ar.currentline = line;
 352     /* Top frame, nextframe = NULL. */
 353     ar.i_ci = (int)((L->base-1) - tvref(L->stack));
 354     lj_state_checkstack(L, 1+LUA_MINSTACK);
 355     hook_enter(g);
 356     hookf(L, &ar);
 357     lua_assert(hook_active(g));
 358     hook_leave(g);
 359   }
 360 }
 361
 362 /* -- Dispatch callbacks -------------------------------------------------- */
 363
 364 /* Calculate number of used stack slots in the current frame. */
 365 static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
 366 {
 367   BCIns ins = pc[-1];
 368   if (bc_op(ins) == BC_UCLO)
 369     ins = pc[bc_j(ins)];
 370   switch (bc_op(ins)) {
 371   case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1;
 372   case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
 373   case BC_TSETM: return bc_a(ins) + nres-1;
 374   default: return pt->framesize;
 375   }
 376 }
 377
 378 /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
 379 void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
 380 {
 381   ERRNO_SAVE
 382   GCfunc *fn = curr_func(L);
 383   GCproto *pt = funcproto(fn);
 384   void *cf = cframe_raw(L->cframe);
 385   const BCIns *oldpc = cframe_pc(cf);
 386   global_State *g = G(L);
 387   BCReg slots;
 388   setcframe_pc(cf, pc);
 389   slots = cur_topslot(pt, pc, cframe_multres_n(cf));
 390   L->top = L->base + slots;  /* Fix top. */
 391 #if LJ_HASJIT
 392   {
 393     jit_State *J = G2J(g);
 394     if (J->state != LJ_TRACE_IDLE) {
 395 #ifdef LUA_USE_ASSERT
 396       ptrdiff_t delta = L->top - L->base;
 397 #endif
 398       J->L = L;
 399       lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 400       lua_assert(L->top - L->base == delta);
 401     }
 402   }
 403 #endif
 404   if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
 405     g->hookcount = g->hookcstart;
 406     callhook(L, LUA_HOOKCOUNT, -1);
 407     L->top = L->base + slots;  /* Fix top again. */
 408   }
 409   if ((g->hookmask & LUA_MASKLINE)) {
 410     BCPos npc = proto_bcpos(pt, pc) - 1;
 411     BCPos opc = proto_bcpos(pt, oldpc) - 1;
 412     BCLine line = lj_debug_line(pt, npc);
 413     if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
 414       callhook(L, LUA_HOOKLINE, line);
 415       L->top = L->base + slots;  /* Fix top again. */
 416     }
 417   }
 418   if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
 419     callhook(L, LUA_HOOKRET, -1);
 420   ERRNO_RESTORE
 421 }
 422
 423 /* Initialize call. Ensure stack space and return # of missing parameters. */
 424 static int call_init(lua_State *L, GCfunc *fn)
 425 {
 426   if (isluafunc(fn)) {
 427     GCproto *pt = funcproto(fn);
 428     int numparams = pt->numparams;
 429     int gotparams = (int)(L->top - L->base);
 430     int need = pt->framesize;
 431     if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
 432     lj_state_checkstack(L, (MSize)need);
 433     numparams -= gotparams;
 434     return numparams >= 0 ? numparams : 0;
 435   } else {
 436     lj_state_checkstack(L, LUA_MINSTACK);
 437     return 0;
 438   }
 439 }
 440
 441 /* Call dispatch. Used by call hooks, hot calls or when recording. */
 442 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 443 {
 444   ERRNO_SAVE
 445   GCfunc *fn = curr_func(L);
 446   BCOp op;
 447   global_State *g = G(L);
 448 #if LJ_HASJIT
 449   jit_State *J = G2J(g);
 450 #endif
 451   int missing = call_init(L, fn);
 452 #if LJ_HASJIT
 453   J->L = L;
 454   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
 455 #ifdef LUA_USE_ASSERT
 456     ptrdiff_t delta = L->top - L->base;
 457 #endif
 458     pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
 459     lj_trace_hot(J, pc);
 460     lua_assert(L->top - L->base == delta);
 461     goto out;
 462   } else if (J->state != LJ_TRACE_IDLE &&
 463              !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 464 #ifdef LUA_USE_ASSERT
 465     ptrdiff_t delta = L->top - L->base;
 466 #endif
 467     /* Record the FUNC* bytecodes, too. */
 468     lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
 469     lua_assert(L->top - L->base == delta);
 470   }
 471 #endif
 472   if ((g->hookmask & LUA_MASKCALL)) {
 473     int i;
 474     for (i = 0; i < missing; i++)  /* Add missing parameters. */
 475       setnilV(L->top++);
 476     callhook(L, LUA_HOOKCALL, -1);
 477     /* Preserve modifications of missing parameters by lua_setlocal(). */
 478     while (missing-- > 0 && tvisnil(L->top - 1))
 479       L->top--;
 480   }
 481 #if LJ_HASJIT
 482 out:
 483 #endif
 484   op = bc_op(pc[-1]);  /* Get FUNC* op. */
 485 #if LJ_HASJIT
 486   /* Use the non-hotcounting variants if JIT is off or while recording. */
 487   if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
 488       (op == BC_FUNCF || op == BC_FUNCV))
 489     op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
 490 #endif
 491   ERRNO_RESTORE
 492   return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
 493 }
 494