2 ** Common definitions for the JIT compiler.
3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
12 /* JIT engine flags. */
13 #define JIT_F_ON 0x00000001
15 /* CPU-specific JIT engine flags. */
16 #if LJ_TARGET_X86ORX64
17 #define JIT_F_SSE2 0x00000010
18 #define JIT_F_SSE3 0x00000020
19 #define JIT_F_SSE4_1 0x00000040
20 #define JIT_F_PREFER_IMUL 0x00000080
21 #define JIT_F_LEA_AGU 0x00000100
23 /* Names for the CPU-specific flags. Must match the order above. */
24 #define JIT_F_CPU_FIRST JIT_F_SSE2
25 #define JIT_F_CPUSTRING "\4SSE2\4SSE3\6SSE4.1\3AMD\4ATOM"
27 #define JIT_F_ARMV6_ 0x00000010
28 #define JIT_F_ARMV6T2_ 0x00000020
29 #define JIT_F_ARMV7 0x00000040
30 #define JIT_F_VFPV2 0x00000080
31 #define JIT_F_VFPV3 0x00000100
33 #define JIT_F_ARMV6 (JIT_F_ARMV6_|JIT_F_ARMV6T2_|JIT_F_ARMV7)
34 #define JIT_F_ARMV6T2 (JIT_F_ARMV6T2_|JIT_F_ARMV7)
35 #define JIT_F_VFP (JIT_F_VFPV2|JIT_F_VFPV3)
37 /* Names for the CPU-specific flags. Must match the order above. */
38 #define JIT_F_CPU_FIRST JIT_F_ARMV6_
39 #define JIT_F_CPUSTRING "\5ARMv6\7ARMv6T2\5ARMv7\5VFPv2\5VFPv3"
41 #define JIT_F_SQRT 0x00000010
42 #define JIT_F_ROUND 0x00000020
44 /* Names for the CPU-specific flags. Must match the order above. */
45 #define JIT_F_CPU_FIRST JIT_F_SQRT
46 #define JIT_F_CPUSTRING "\4SQRT\5ROUND"
48 #define JIT_F_MIPS32R2 0x00000010
50 /* Names for the CPU-specific flags. Must match the order above. */
51 #define JIT_F_CPU_FIRST JIT_F_MIPS32R2
52 #define JIT_F_CPUSTRING "\010MIPS32R2"
54 #define JIT_F_CPU_FIRST 0
55 #define JIT_F_CPUSTRING ""
58 /* Optimization flags. */
59 #define JIT_F_OPT_MASK 0x0fff0000
61 #define JIT_F_OPT_FOLD 0x00010000
62 #define JIT_F_OPT_CSE 0x00020000
63 #define JIT_F_OPT_DCE 0x00040000
64 #define JIT_F_OPT_FWD 0x00080000
65 #define JIT_F_OPT_DSE 0x00100000
66 #define JIT_F_OPT_NARROW 0x00200000
67 #define JIT_F_OPT_LOOP 0x00400000
68 #define JIT_F_OPT_ABC 0x00800000
69 #define JIT_F_OPT_SINK 0x01000000
70 #define JIT_F_OPT_FUSE 0x02000000
72 /* Optimizations names for -O. Must match the order above. */
73 #define JIT_F_OPT_FIRST JIT_F_OPT_FOLD
74 #define JIT_F_OPTSTRING \
75 "\4fold\3cse\3dce\3fwd\3dse\6narrow\4loop\3abc\4sink\4fuse"
77 /* Optimization levels set a fixed combination of flags. */
79 #define JIT_F_OPT_1 (JIT_F_OPT_FOLD|JIT_F_OPT_CSE|JIT_F_OPT_DCE)
80 #define JIT_F_OPT_2 (JIT_F_OPT_1|JIT_F_OPT_NARROW|JIT_F_OPT_LOOP)
81 #define JIT_F_OPT_3 (JIT_F_OPT_2|\
82 JIT_F_OPT_FWD|JIT_F_OPT_DSE|JIT_F_OPT_ABC|JIT_F_OPT_SINK|JIT_F_OPT_FUSE)
83 #define JIT_F_OPT_DEFAULT JIT_F_OPT_3
85 #if LJ_TARGET_WINDOWS || LJ_64
86 /* See: http://blogs.msdn.com/oldnewthing/archive/2003/10/08/55239.aspx */
87 #define JIT_P_sizemcode_DEFAULT 64
89 /* Could go as low as 4K, but the mmap() overhead would be rather high. */
90 #define JIT_P_sizemcode_DEFAULT 32
93 /* Optimization parameters and their defaults. Length is a char in octal! */
94 #define JIT_PARAMDEF(_) \
95 _(\010, maxtrace, 1000) /* Max. # of traces in cache. */ \
96 _(\011, maxrecord, 4000) /* Max. # of recorded IR instructions. */ \
97 _(\012, maxirconst, 500) /* Max. # of IR constants of a trace. */ \
98 _(\007, maxside, 100) /* Max. # of side traces of a root trace. */ \
99 _(\007, maxsnap, 500) /* Max. # of snapshots for a trace. */ \
100 _(\011, minstitch, 0) /* Min. # of IR ins for a stitched trace. */ \
102 _(\007, hotloop, 56) /* # of iter. to detect a hot loop/call. */ \
103 _(\007, hotexit, 10) /* # of taken exits to start a side trace. */ \
104 _(\007, tryside, 4) /* # of attempts to compile a side trace. */ \
106 _(\012, instunroll, 4) /* Max. unroll for instable loops. */ \
107 _(\012, loopunroll, 15) /* Max. unroll for loop ops in side traces. */ \
108 _(\012, callunroll, 3) /* Max. unroll for recursive calls. */ \
109 _(\011, recunroll, 2) /* Min. unroll for true recursion. */ \
111 /* Size of each machine code area (in KBytes). */ \
112 _(\011, sizemcode, JIT_P_sizemcode_DEFAULT) \
113 /* Max. total size of all machine code areas (in KBytes). */ \
114 _(\010, maxmcode, 512) \
118 #define JIT_PARAMENUM(len, name, value) JIT_P_##name,
119 JIT_PARAMDEF(JIT_PARAMENUM
)
124 #define JIT_PARAMSTR(len, name, value) #len #name
125 #define JIT_P_STRING JIT_PARAMDEF(JIT_PARAMSTR)
127 /* Trace compiler state. */
129 LJ_TRACE_IDLE
, /* Trace compiler idle. */
130 LJ_TRACE_ACTIVE
= 0x10,
131 LJ_TRACE_RECORD
, /* Bytecode recording active. */
132 LJ_TRACE_START
, /* New trace started. */
133 LJ_TRACE_END
, /* End of trace. */
134 LJ_TRACE_ASM
, /* Assemble trace. */
135 LJ_TRACE_ERR
/* Trace aborted with error. */
138 /* Post-processing action. */
140 LJ_POST_NONE
, /* No action. */
141 LJ_POST_FIXCOMP
, /* Fixup comparison and emit pending guard. */
142 LJ_POST_FIXGUARD
, /* Fixup and emit pending guard. */
143 LJ_POST_FIXGUARDSNAP
, /* Fixup and emit pending guard and snapshot. */
144 LJ_POST_FIXBOOL
, /* Fixup boolean result. */
145 LJ_POST_FIXCONST
, /* Fixup constant results. */
146 LJ_POST_FFRETRY
/* Suppress recording of retried fast functions. */
149 /* Machine code type. */
150 #if LJ_TARGET_X86ORX64
151 typedef uint8_t MCode
;
153 typedef uint32_t MCode
;
156 /* Stack snapshot header. */
157 typedef struct SnapShot
{
158 uint16_t mapofs
; /* Offset into snapshot map. */
159 IRRef1 ref
; /* First IR ref for this snapshot. */
160 uint8_t nslots
; /* Number of valid slots. */
161 uint8_t topslot
; /* Maximum frame extent. */
162 uint8_t nent
; /* Number of compressed entries. */
163 uint8_t count
; /* Count of taken exits for this snapshot. */
166 #define SNAPCOUNT_DONE 255 /* Already compiled and linked a side trace. */
168 /* Compressed snapshot entry. */
169 typedef uint32_t SnapEntry
;
171 #define SNAP_FRAME 0x010000 /* Frame slot. */
172 #define SNAP_CONT 0x020000 /* Continuation slot. */
173 #define SNAP_NORESTORE 0x040000 /* No need to restore slot. */
174 #define SNAP_SOFTFPNUM 0x080000 /* Soft-float number. */
175 LJ_STATIC_ASSERT(SNAP_FRAME
== TREF_FRAME
);
176 LJ_STATIC_ASSERT(SNAP_CONT
== TREF_CONT
);
178 #define SNAP(slot, flags, ref) (((SnapEntry)(slot) << 24) + (flags) + (ref))
179 #define SNAP_TR(slot, tr) \
180 (((SnapEntry)(slot) << 24) + ((tr) & (TREF_CONT|TREF_FRAME|TREF_REFMASK)))
181 #define SNAP_MKPC(pc) ((SnapEntry)u32ptr(pc))
182 #define SNAP_MKFTSZ(ftsz) ((SnapEntry)(ftsz))
183 #define snap_ref(sn) ((sn) & 0xffff)
184 #define snap_slot(sn) ((BCReg)((sn) >> 24))
185 #define snap_isframe(sn) ((sn) & SNAP_FRAME)
186 #define snap_pc(sn) ((const BCIns *)(uintptr_t)(sn))
187 #define snap_setref(sn, ref) (((sn) & (0xffff0000&~SNAP_NORESTORE)) | (ref))
189 /* Snapshot and exit numbers. */
190 typedef uint32_t SnapNo
;
191 typedef uint32_t ExitNo
;
194 typedef uint32_t TraceNo
; /* Used to pass around trace numbers. */
195 typedef uint16_t TraceNo1
; /* Stored trace number. */
197 /* Type of link. ORDER LJ_TRLINK */
199 LJ_TRLINK_NONE
, /* Incomplete trace. No link, yet. */
200 LJ_TRLINK_ROOT
, /* Link to other root trace. */
201 LJ_TRLINK_LOOP
, /* Loop to same trace. */
202 LJ_TRLINK_TAILREC
, /* Tail-recursion. */
203 LJ_TRLINK_UPREC
, /* Up-recursion. */
204 LJ_TRLINK_DOWNREC
, /* Down-recursion. */
205 LJ_TRLINK_INTERP
, /* Fallback to interpreter. */
206 LJ_TRLINK_RETURN
, /* Return to interpreter. */
207 LJ_TRLINK_STITCH
/* Trace stitching. */
211 typedef struct GCtrace
{
213 uint8_t topslot
; /* Top stack slot already checked to be allocated. */
214 uint8_t linktype
; /* Type of link. */
215 IRRef nins
; /* Next IR instruction. Biased with REF_BIAS. */
217 IRIns
*ir
; /* IR instructions/constants. Biased with REF_BIAS. */
218 IRRef nk
; /* Lowest IR constant. Biased with REF_BIAS. */
219 uint16_t nsnap
; /* Number of snapshots. */
220 uint16_t nsnapmap
; /* Number of snapshot map elements. */
221 SnapShot
*snap
; /* Snapshot array. */
222 SnapEntry
*snapmap
; /* Snapshot map. */
223 GCRef startpt
; /* Starting prototype. */
224 MRef startpc
; /* Bytecode PC of starting instruction. */
225 BCIns startins
; /* Original bytecode of starting instruction. */
226 MSize szmcode
; /* Size of machine code. */
227 MCode
*mcode
; /* Start of machine code. */
228 MSize mcloop
; /* Offset of loop start in machine code. */
229 uint16_t nchild
; /* Number of child traces (root trace only). */
230 uint16_t spadjust
; /* Stack pointer adjustment (offset in bytes). */
231 TraceNo1 traceno
; /* Trace number. */
232 TraceNo1 link
; /* Linked trace (or self for loops). */
233 TraceNo1 root
; /* Root trace of side trace (or 0 for root traces). */
234 TraceNo1 nextroot
; /* Next root trace for same prototype. */
235 TraceNo1 nextside
; /* Next side trace of same root trace. */
236 uint8_t sinktags
; /* Trace has SINK tags. */
238 #ifdef LUAJIT_USE_GDBJIT
239 void *gdbjit_entry
; /* GDB JIT entry. */
243 #define gco2trace(o) check_exp((o)->gch.gct == ~LJ_TTRACE, (GCtrace *)(o))
244 #define traceref(J, n) \
245 check_exp((n)>0 && (MSize)(n)<J->sizetrace, (GCtrace *)gcref(J->trace[(n)]))
247 LJ_STATIC_ASSERT(offsetof(GChead
, gclist
) == offsetof(GCtrace
, gclist
));
249 static LJ_AINLINE MSize
snap_nextofs(GCtrace
*T
, SnapShot
*snap
)
251 if (snap
+1 == &T
->snap
[T
->nsnap
])
254 return (snap
+1)->mapofs
;
257 /* Round-robin penalty cache for bytecodes leading to aborted traces. */
258 typedef struct HotPenalty
{
259 MRef pc
; /* Starting bytecode PC. */
260 uint16_t val
; /* Penalty value, i.e. hotcount start. */
261 uint16_t reason
; /* Abort reason (really TraceErr). */
264 #define PENALTY_SLOTS 64 /* Penalty cache slot. Must be a power of 2. */
265 #define PENALTY_MIN (36*2) /* Minimum penalty value. */
266 #define PENALTY_MAX 60000 /* Maximum penalty value. */
267 #define PENALTY_RNDBITS 4 /* # of random bits to add to penalty value. */
269 /* Round-robin backpropagation cache for narrowing conversions. */
270 typedef struct BPropEntry
{
271 IRRef1 key
; /* Key: original reference. */
272 IRRef1 val
; /* Value: reference after conversion. */
273 IRRef mode
; /* Mode for this entry (currently IRCONV_*). */
276 /* Number of slots for the backpropagation cache. Must be a power of 2. */
277 #define BPROP_SLOTS 16
279 /* Scalar evolution analysis cache. */
280 typedef struct ScEvEntry
{
281 MRef pc
; /* Bytecode PC of FORI. */
282 IRRef1 idx
; /* Index reference. */
283 IRRef1 start
; /* Constant start reference. */
284 IRRef1 stop
; /* Constant stop reference. */
285 IRRef1 step
; /* Constant step reference. */
286 IRType1 t
; /* Scalar type. */
287 uint8_t dir
; /* Direction. 1: +, 0: -. */
290 /* 128 bit SIMD constants. */
297 /* Get 16 byte aligned pointer to SIMD constant. */
298 #define LJ_KSIMD(J, n) \
299 ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15))
301 /* Set/reset flag to activate the SPLIT pass for the current trace. */
302 #if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
303 #define lj_needsplit(J) (J->needsplit = 1)
304 #define lj_resetsplit(J) (J->needsplit = 0)
306 #define lj_needsplit(J) UNUSED(J)
307 #define lj_resetsplit(J) UNUSED(J)
310 /* Fold state is used to fold instructions on-the-fly. */
311 typedef struct FoldState
{
312 IRIns ins
; /* Currently emitted instruction. */
313 IRIns left
; /* Instruction referenced by left operand. */
314 IRIns right
; /* Instruction referenced by right operand. */
317 /* JIT compiler state. */
318 typedef struct jit_State
{
319 GCtrace cur
; /* Current trace. */
321 lua_State
*L
; /* Current Lua state. */
322 const BCIns
*pc
; /* Current PC. */
323 GCfunc
*fn
; /* Current function. */
324 GCproto
*pt
; /* Current prototype. */
325 TRef
*base
; /* Current frame base, points into J->slots. */
327 uint32_t flags
; /* JIT engine flags. */
328 BCReg maxslot
; /* Relative to baseslot. */
329 BCReg baseslot
; /* Current frame base, offset into J->slots. */
331 uint8_t mergesnap
; /* Allowed to merge with next snapshot. */
332 uint8_t needsnap
; /* Need snapshot before recording next bytecode. */
333 IRType1 guardemit
; /* Accumulated IRT_GUARD for emitted instructions. */
334 uint8_t bcskip
; /* Number of bytecode instructions to skip. */
336 FoldState fold
; /* Fold state. */
338 const BCIns
*bc_min
; /* Start of allowed bytecode range for root trace. */
339 MSize bc_extent
; /* Extent of the range. */
341 TraceState state
; /* Trace compiler state. */
343 int32_t instunroll
; /* Unroll counter for instable loops. */
344 int32_t loopunroll
; /* Unroll counter for loop ops in side traces. */
345 int32_t tailcalled
; /* Number of successive tailcalls. */
346 int32_t framedepth
; /* Current frame depth. */
347 int32_t retdepth
; /* Return frame depth (count of RETF). */
349 MRef k64
; /* Pointer to chained array of 64 bit constants. */
350 TValue ksimd
[LJ_KSIMD__MAX
*2+1]; /* 16 byte aligned SIMD constants. */
352 IRIns
*irbuf
; /* Temp. IR instruction buffer. Biased with REF_BIAS. */
353 IRRef irtoplim
; /* Upper limit of instuction buffer (biased). */
354 IRRef irbotlim
; /* Lower limit of instuction buffer (biased). */
355 IRRef loopref
; /* Last loop reference or ref of final LOOP (or 0). */
357 MSize sizesnap
; /* Size of temp. snapshot buffer. */
358 SnapShot
*snapbuf
; /* Temp. snapshot buffer. */
359 SnapEntry
*snapmapbuf
; /* Temp. snapshot map buffer. */
360 MSize sizesnapmap
; /* Size of temp. snapshot map buffer. */
362 PostProc postproc
; /* Required post-processing after execution. */
363 #if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
364 int needsplit
; /* Need SPLIT pass. */
367 GCRef
*trace
; /* Array of traces. */
368 TraceNo freetrace
; /* Start of scan for next free trace. */
369 MSize sizetrace
; /* Size of trace array. */
371 IRRef1 chain
[IR__MAX
]; /* IR instruction skip-list chain anchors. */
372 TRef slot
[LJ_MAX_JSLOTS
+LJ_STACK_EXTRA
]; /* Stack slot map. */
374 int32_t param
[JIT_P__MAX
]; /* JIT engine parameters. */
376 MCode
*exitstubgroup
[LJ_MAX_EXITSTUBGR
]; /* Exit stub group addresses. */
378 HotPenalty penalty
[PENALTY_SLOTS
]; /* Penalty slots. */
379 uint32_t penaltyslot
; /* Round-robin index into penalty slots. */
380 uint32_t prngstate
; /* PRNG state. */
382 BPropEntry bpropcache
[BPROP_SLOTS
]; /* Backpropagation cache slots. */
383 uint32_t bpropslot
; /* Round-robin index into bpropcache slots. */
385 ScEvEntry scev
; /* Scalar evolution analysis cache slots. */
387 const BCIns
*startpc
; /* Bytecode PC of starting instruction. */
388 TraceNo parent
; /* Parent of current side trace (0 for root traces). */
389 ExitNo exitno
; /* Exit number in parent of current side trace. */
391 BCIns
*patchpc
; /* PC for pending re-patch. */
392 BCIns patchins
; /* Instruction for pending re-patch. */
394 int mcprot
; /* Protection of current mcode area. */
395 MCode
*mcarea
; /* Base of current mcode area. */
396 MCode
*mctop
; /* Top of current mcode area. */
397 MCode
*mcbot
; /* Bottom of current mcode area. */
398 size_t szmcarea
; /* Size of current mcode area. */
399 size_t szallmcarea
; /* Total size of all allocated mcode areas. */
401 TValue errinfo
; /* Additional info element for trace errors. */
404 GCproto
*prev_pt
; /* Previous prototype. */
405 BCLine prev_line
; /* Previous line. */
406 int prof_mode
; /* Profiling mode: 0, 'f', 'l'. */
410 LJ_ALIGN(16) /* For DISPATCH-relative addresses in assembler part. */
414 /* Trivial PRNG e.g. used for penalty randomization. */
415 static LJ_AINLINE
uint32_t LJ_PRNG_BITS(jit_State
*J
, int bits
)
417 /* Yes, this LCG is very weak, but that doesn't matter for our use case. */
418 J
->prngstate
= J
->prngstate
* 1103515245 + 12345;
419 return J
->prngstate
>> (32-bits
);