src/lj_dispatch.h

   1 /*
   2 ** Instruction dispatch handling.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #ifndef _LJ_DISPATCH_H
   7 #define _LJ_DISPATCH_H
   8
   9 #include "lj_obj.h"
  10 #include "lj_bc.h"
  11 #if LJ_HASJIT
  12 #include "lj_jit.h"
  13 #endif
  14
  15 #if LJ_TARGET_MIPS
  16 /* Need our own global offset table for the dreaded MIPS calling conventions. */
  17
  18 #ifndef _LJ_VM_H
  19 LJ_ASMF int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b);
  20 #endif
  21
  22 #if LJ_SOFTFP
  23 #ifndef _LJ_IRCALL_H
  24 extern double __adddf3(double a, double b);
  25 extern double __subdf3(double a, double b);
  26 extern double __muldf3(double a, double b);
  27 extern double __divdf3(double a, double b);
  28 #endif
  29 #define SFGOTDEF(_)     _(sqrt) _(__adddf3) _(__subdf3) _(__muldf3) _(__divdf3)
  30 #else
  31 #define SFGOTDEF(_)
  32 #endif
  33 #if LJ_HASJIT
  34 #define JITGOTDEF(_)    _(lj_err_trace) _(lj_trace_exit) _(lj_trace_hot)
  35 #else
  36 #define JITGOTDEF(_)
  37 #endif
  38 #if LJ_HASFFI
  39 #define FFIGOTDEF(_) \
  40   _(lj_meta_equal_cd) _(lj_ccallback_enter) _(lj_ccallback_leave)
  41 #else
  42 #define FFIGOTDEF(_)
  43 #endif
  44 #define GOTDEF(_) \
  45   _(floor) _(ceil) _(trunc) _(log) _(log10) _(exp) _(sin) _(cos) _(tan) \
  46   _(asin) _(acos) _(atan) _(sinh) _(cosh) _(tanh) _(frexp) _(modf) _(atan2) \
  47   _(pow) _(fmod) _(ldexp) _(lj_vm_modi) \
  48   _(lj_dispatch_call) _(lj_dispatch_ins) _(lj_dispatch_stitch) \
  49   _(lj_dispatch_profile) _(lj_err_throw) \
  50   _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \
  51   _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \
  52   _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \
  53   _(lj_meta_for) _(lj_meta_istype) _(lj_meta_len) _(lj_meta_tget) \
  54   _(lj_meta_tset) _(lj_state_growstack) _(lj_strfmt_number) \
  55   _(lj_str_new) _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) \
  56   _(lj_tab_new) _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
  57   _(lj_tab_setinth) _(lj_buf_putstr_reverse) _(lj_buf_putstr_lower) \
  58   _(lj_buf_putstr_upper) _(lj_buf_tostr) \
  59   JITGOTDEF(_) FFIGOTDEF(_) SFGOTDEF(_)
  60
  61 enum {
  62 #define GOTENUM(name) LJ_GOT_##name,
  63 GOTDEF(GOTENUM)
  64 #undef GOTENUM
  65   LJ_GOT__MAX
  66 };
  67 #endif
  68
  69 /* Type of hot counter. Must match the code in the assembler VM. */
  70 /* 16 bits are sufficient. Only 0.0015% overhead with maximum slot penalty. */
  71 typedef uint16_t HotCount;
  72
  73 /* Number of hot counter hash table entries (must be a power of two). */
  74 #define HOTCOUNT_SIZE           64
  75 #define HOTCOUNT_PCMASK         ((HOTCOUNT_SIZE-1)*sizeof(HotCount))
  76
  77 /* Hotcount decrements. */
  78 #define HOTCOUNT_LOOP           2
  79 #define HOTCOUNT_CALL           1
  80
  81 /* This solves a circular dependency problem -- bump as needed. Sigh. */
  82 #define GG_NUM_ASMFF    57
  83
  84 #define GG_LEN_DDISP    (BC__MAX + GG_NUM_ASMFF)
  85 #define GG_LEN_SDISP    BC_FUNCF
  86 #define GG_LEN_DISP     (GG_LEN_DDISP + GG_LEN_SDISP)
  87
  88 /* Global state, main thread and extra fields are allocated together. */
  89 typedef struct GG_State {
  90   lua_State L;                          /* Main thread. */
  91   global_State g;                       /* Global state. */
  92 #if LJ_TARGET_ARM && !LJ_TARGET_NX
  93   /* Make g reachable via K12 encoded DISPATCH-relative addressing. */
  94   uint8_t align1[(16-sizeof(global_State))&15];
  95 #endif
  96 #if LJ_TARGET_MIPS
  97   ASMFunction got[LJ_GOT__MAX];         /* Global offset table. */
  98 #endif
  99 #if LJ_HASJIT
 100   jit_State J;                          /* JIT state. */
 101   HotCount hotcount[HOTCOUNT_SIZE];     /* Hot counters. */
 102 #if LJ_TARGET_ARM && !LJ_TARGET_NX
 103   /* Ditto for J. */
 104   uint8_t align2[(16-sizeof(jit_State)-sizeof(HotCount)*HOTCOUNT_SIZE)&15];
 105 #endif
 106 #endif
 107   ASMFunction dispatch[GG_LEN_DISP];    /* Instruction dispatch tables. */
 108   BCIns bcff[GG_NUM_ASMFF];             /* Bytecode for ASM fast functions. */
 109 } GG_State;
 110
 111 #define GG_OFS(field)   ((int)offsetof(GG_State, field))
 112 #define G2GG(gl)        ((GG_State *)((char *)(gl) - GG_OFS(g)))
 113 #define J2GG(j)         ((GG_State *)((char *)(j) - GG_OFS(J)))
 114 #define L2GG(L)         (G2GG(G(L)))
 115 #define J2G(J)          (&J2GG(J)->g)
 116 #define G2J(gl)         (&G2GG(gl)->J)
 117 #define L2J(L)          (&L2GG(L)->J)
 118 #define GG_G2J          (GG_OFS(J) - GG_OFS(g))
 119 #define GG_G2DISP       (GG_OFS(dispatch) - GG_OFS(g))
 120 #define GG_DISP2G       (GG_OFS(g) - GG_OFS(dispatch))
 121 #define GG_DISP2J       (GG_OFS(J) - GG_OFS(dispatch))
 122 #define GG_DISP2HOT     (GG_OFS(hotcount) - GG_OFS(dispatch))
 123 #define GG_DISP2STATIC  (GG_LEN_DDISP*(int)sizeof(ASMFunction))
 124
 125 #define hotcount_get(gg, pc) \
 126   (gg)->hotcount[(u32ptr(pc)>>2) & (HOTCOUNT_SIZE-1)]
 127 #define hotcount_set(gg, pc, val) \
 128   (hotcount_get((gg), (pc)) = (HotCount)(val))
 129
 130 /* Dispatch table management. */
 131 LJ_FUNC void lj_dispatch_init(GG_State *GG);
 132 #if LJ_HASJIT
 133 LJ_FUNC void lj_dispatch_init_hotcount(global_State *g);
 134 #endif
 135 LJ_FUNC void lj_dispatch_update(global_State *g);
 136
 137 /* Instruction dispatch callback for hooks or when recording. */
 138 LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc);
 139 LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns*pc);
 140 #if LJ_HASJIT
 141 LJ_FUNCA void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc);
 142 #endif
 143 #if LJ_HASPROFILE
 144 LJ_FUNCA void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc);
 145 #endif
 146
 147 #if LJ_HASFFI && !defined(_BUILDVM_H)
 148 /* Save/restore errno and GetLastError() around hooks, exits and recording. */
 149 #include <errno.h>
 150 #if LJ_TARGET_WINDOWS
 151 #define WIN32_LEAN_AND_MEAN
 152 #include <windows.h>
 153 #define ERRNO_SAVE      int olderr = errno; DWORD oldwerr = GetLastError();
 154 #define ERRNO_RESTORE   errno = olderr; SetLastError(oldwerr);
 155 #else
 156 #define ERRNO_SAVE      int olderr = errno;
 157 #define ERRNO_RESTORE   errno = olderr;
 158 #endif
 159 #else
 160 #define ERRNO_SAVE
 161 #define ERRNO_RESTORE
 162 #endif
 163
 164 #endif