2 ** Trace recorder (bytecode -> SSA IR).
3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
23 #include "lj_record.h"
24 #include "lj_ffrecord.h"
26 #include "lj_dispatch.h"
29 /* Some local macros to save typing. Undef'd at the end. */
30 #define IR(ref) (&J->cur.ir[(ref)])
32 /* Pass IR on to next optimization in chain (FOLD). */
33 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
35 /* Emit raw IR without passing through optimizations. */
36 #define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
38 /* -- Sanity checks ------------------------------------------------------- */
41 /* Sanity check the whole IR -- sloooow. */
42 static void rec_check_ir(jit_State
*J
)
44 IRRef i
, nins
= J
->cur
.nins
, nk
= J
->cur
.nk
;
45 lua_assert(nk
<= REF_BIAS
&& nins
>= REF_BIAS
&& nins
< 65536);
46 for (i
= nins
-1; i
>= nk
; i
--) {
48 uint32_t mode
= lj_ir_mode
[ir
->o
];
51 switch (irm_op1(mode
)) {
52 case IRMnone
: lua_assert(op1
== 0); break;
53 case IRMref
: lua_assert(op1
>= nk
);
54 lua_assert(i
>= REF_BIAS
? op1
< i
: op1
> i
); break;
56 case IRMcst
: lua_assert(i
< REF_BIAS
); continue;
58 switch (irm_op2(mode
)) {
59 case IRMnone
: lua_assert(op2
== 0); break;
60 case IRMref
: lua_assert(op2
>= nk
);
61 lua_assert(i
>= REF_BIAS
? op2
< i
: op2
> i
); break;
63 case IRMcst
: lua_assert(0); break;
66 lua_assert(ir
->prev
>= nk
);
67 lua_assert(i
>= REF_BIAS
? ir
->prev
< i
: ir
->prev
> i
);
68 lua_assert(IR(ir
->prev
)->o
== ir
->o
);
73 /* Compare stack slots and frames of the recorder and the VM. */
74 static void rec_check_slots(jit_State
*J
)
76 BCReg s
, nslots
= J
->baseslot
+ J
->maxslot
;
78 cTValue
*base
= J
->L
->base
- J
->baseslot
;
79 lua_assert(J
->baseslot
>= 1 && J
->baseslot
< LJ_MAX_JSLOTS
);
80 lua_assert(J
->baseslot
== 1 || (J
->slot
[J
->baseslot
-1] & TREF_FRAME
));
81 lua_assert(nslots
< LJ_MAX_JSLOTS
);
82 for (s
= 0; s
< nslots
; s
++) {
85 cTValue
*tv
= &base
[s
];
86 IRRef ref
= tref_ref(tr
);
88 lua_assert(ref
>= J
->cur
.nk
&& ref
< J
->cur
.nins
);
90 lua_assert(irt_t(ir
->t
) == tref_t(tr
));
92 lua_assert(tref_isfunc(tr
));
93 } else if ((tr
& TREF_FRAME
)) {
94 GCfunc
*fn
= gco2func(frame_gc(tv
));
95 BCReg delta
= (BCReg
)(tv
- frame_prev(tv
));
96 lua_assert(tref_isfunc(tr
));
97 if (tref_isk(tr
)) lua_assert(fn
== ir_kfunc(ir
));
98 lua_assert(s
> delta
? (J
->slot
[s
-delta
] & TREF_FRAME
) : (s
== delta
));
100 } else if ((tr
& TREF_CONT
)) {
101 lua_assert(ir_kptr(ir
) == gcrefp(tv
->gcr
, void));
102 lua_assert((J
->slot
[s
+1] & TREF_FRAME
));
106 lua_assert(tref_isnumber(tr
)); /* Could be IRT_INT etc., too. */
108 lua_assert(itype2irt(tv
) == tref_type(tr
));
109 if (tref_isk(tr
)) { /* Compare constants. */
111 lj_ir_kvalue(J
->L
, &tvk
, ir
);
112 if (!(tvisnum(&tvk
) && tvisnan(&tvk
)))
113 lua_assert(lj_obj_equal(tv
, &tvk
));
115 lua_assert(tvisnum(tv
) && tvisnan(tv
));
120 lua_assert(J
->framedepth
== depth
);
124 /* -- Type handling and specialization ------------------------------------ */
126 /* Note: these functions return tagged references (TRef). */
128 /* Specialize a slot to a specific type. Note: slot can be negative! */
129 static TRef
sloadt(jit_State
*J
, int32_t slot
, IRType t
, int mode
)
131 /* Caller may set IRT_GUARD in t. */
132 TRef ref
= emitir_raw(IRT(IR_SLOAD
, t
), (int32_t)J
->baseslot
+slot
, mode
);
137 /* Specialize a slot to the runtime type. Note: slot can be negative! */
138 static TRef
sload(jit_State
*J
, int32_t slot
)
140 IRType t
= itype2irt(&J
->L
->base
[slot
]);
141 TRef ref
= emitir_raw(IRTG(IR_SLOAD
, t
), (int32_t)J
->baseslot
+slot
,
143 if (irtype_ispri(t
)) ref
= TREF_PRI(t
); /* Canonicalize primitive refs. */
148 /* Get TRef from slot. Load slot and specialize if not done already. */
149 #define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
151 /* Get TRef for current function. */
152 static TRef
getcurrf(jit_State
*J
)
156 lua_assert(J
->baseslot
== 1);
157 return sloadt(J
, -1, IRT_FUNC
, IRSLOAD_READONLY
);
160 /* Compare for raw object equality.
161 ** Returns 0 if the objects are the same.
162 ** Returns 1 if they are different, but the same type.
163 ** Returns 2 for two different types.
164 ** Comparisons between primitives always return 1 -- no caller cares about it.
166 int lj_record_objcmp(jit_State
*J
, TRef a
, TRef b
, cTValue
*av
, cTValue
*bv
)
168 int diff
= !lj_obj_equal(av
, bv
);
169 if (!tref_isk2(a
, b
)) { /* Shortcut, also handles primitives. */
170 IRType ta
= tref_isinteger(a
) ? IRT_INT
: tref_type(a
);
171 IRType tb
= tref_isinteger(b
) ? IRT_INT
: tref_type(b
);
173 /* Widen mixed number/int comparisons to number/number comparison. */
174 if (ta
== IRT_INT
&& tb
== IRT_NUM
) {
175 a
= emitir(IRTN(IR_CONV
), a
, IRCONV_NUM_INT
);
177 } else if (ta
== IRT_NUM
&& tb
== IRT_INT
) {
178 b
= emitir(IRTN(IR_CONV
), b
, IRCONV_NUM_INT
);
180 return 2; /* Two different types are never equal. */
183 emitir(IRTG(diff
? IR_NE
: IR_EQ
, ta
), a
, b
);
188 /* -- Record loop ops ----------------------------------------------------- */
192 LOOPEV_LEAVE
, /* Loop is left or not entered. */
193 LOOPEV_ENTER
/* Loop is entered. */
196 /* Canonicalize slots: convert integers to numbers. */
197 static void canonicalize_slots(jit_State
*J
)
200 for (s
= J
->baseslot
+J
->maxslot
-1; s
>= 1; s
--) {
201 TRef tr
= J
->slot
[s
];
202 if (tref_isinteger(tr
)) {
203 IRIns
*ir
= IR(tref_ref(tr
));
204 if (!(ir
->o
== IR_SLOAD
&& (ir
->op2
& IRSLOAD_READONLY
)))
205 J
->slot
[s
] = emitir(IRTN(IR_CONV
), tr
, IRCONV_NUM_INT
);
210 /* Stop recording. */
211 static void rec_stop(jit_State
*J
, TraceNo lnk
)
214 J
->cur
.link
= (uint16_t)lnk
;
215 /* Looping back at the same stack level? */
216 if (lnk
== J
->cur
.traceno
&& J
->framedepth
+ J
->retdepth
== 0) {
217 if ((J
->flags
& JIT_F_OPT_LOOP
)) /* Shall we try to create a loop? */
218 goto nocanon
; /* Do not canonicalize or we lose the narrowing. */
219 if (J
->cur
.root
) /* Otherwise ensure we always link to the root trace. */
220 J
->cur
.link
= J
->cur
.root
;
222 canonicalize_slots(J
);
224 /* Note: all loop ops must set J->pc to the following instruction! */
225 lj_snap_add(J
); /* Add loop snapshot. */
227 J
->mergesnap
= 1; /* In case recording continues. */
230 /* Search bytecode backwards for a int/num constant slot initializer. */
231 static TRef
find_kinit(jit_State
*J
, const BCIns
*endpc
, BCReg slot
, IRType t
)
233 /* This algorithm is rather simplistic and assumes quite a bit about
234 ** how the bytecode is generated. It works fine for FORI initializers,
235 ** but it won't necessarily work in other cases (e.g. iterator arguments).
236 ** It doesn't do anything fancy, either (like backpropagating MOVs).
238 const BCIns
*pc
, *startpc
= proto_bc(J
->pt
);
239 for (pc
= endpc
-1; pc
> startpc
; pc
--) {
241 BCOp op
= bc_op(ins
);
242 /* First try to find the last instruction that stores to this slot. */
243 if (bcmode_a(op
) == BCMbase
&& bc_a(ins
) <= slot
) {
244 return 0; /* Multiple results, e.g. from a CALL or KNIL. */
245 } else if (bcmode_a(op
) == BCMdst
&& bc_a(ins
) == slot
) {
246 if (op
== BC_KSHORT
|| op
== BC_KNUM
) { /* Found const. initializer. */
247 /* Now try to verify there's no forward jump across it. */
248 const BCIns
*kpc
= pc
;
249 for (; pc
> startpc
; pc
--)
250 if (bc_op(*pc
) == BC_JMP
) {
251 const BCIns
*target
= pc
+bc_j(*pc
)+1;
252 if (target
> kpc
&& target
<= endpc
)
253 return 0; /* Conditional assignment. */
255 if (op
== BC_KSHORT
) {
256 int32_t k
= (int32_t)(int16_t)bc_d(ins
);
257 return t
== IRT_INT
? lj_ir_kint(J
, k
) : lj_ir_knum(J
, cast_num(k
));
259 lua_Number n
= proto_knum(J
->pt
, bc_d(ins
));
261 int32_t k
= lj_num2int(n
);
262 if (n
== cast_num(k
)) /* -0 is ok here. */
263 return lj_ir_kint(J
, k
);
264 return 0; /* Type mismatch. */
266 return lj_ir_knum(J
, n
);
270 return 0; /* Non-constant initializer. */
273 return 0; /* No assignment to this slot found? */
276 /* Peek before FORI to find a const initializer. Otherwise load from slot. */
277 static TRef
fori_arg(jit_State
*J
, const BCIns
*fori
, BCReg slot
, IRType t
)
279 TRef tr
= J
->base
[slot
];
281 tr
= find_kinit(J
, fori
, slot
, t
);
283 tr
= sloadt(J
, (int32_t)slot
,
284 t
== IRT_INT
? (IRT_INT
|IRT_GUARD
) : t
,
285 t
== IRT_INT
? (IRSLOAD_CONVERT
|IRSLOAD_READONLY
|IRSLOAD_INHERIT
) :
286 (IRSLOAD_READONLY
|IRSLOAD_INHERIT
));
291 /* In-place coercion of FORI arguments. */
292 static lua_Number
for_coerce(jit_State
*J
, TValue
*o
)
294 if (!tvisnum(o
) && !(tvisstr(o
) && lj_str_tonum(strV(o
), o
)))
295 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
299 /* Simulate the runtime behavior of the FOR loop iterator.
300 ** It's important to exactly reproduce the semantics of the interpreter.
302 static LoopEvent
for_iter(jit_State
*J
, IROp
*op
, BCReg ra
, int isforl
)
304 TValue
*forbase
= &J
->L
->base
[ra
];
305 lua_Number stopv
= for_coerce(J
, &forbase
[FORL_STOP
]);
306 lua_Number idxv
= for_coerce(J
, &forbase
[FORL_IDX
]);
307 lua_Number stepv
= for_coerce(J
, &forbase
[FORL_STEP
]);
310 if ((int32_t)forbase
[FORL_STEP
].u32
.hi
>= 0) {
311 if (idxv
<= stopv
) { *op
= IR_LE
; return LOOPEV_ENTER
; }
312 *op
= IR_GT
; return LOOPEV_LEAVE
;
314 if (stopv
<= idxv
) { *op
= IR_GE
; return LOOPEV_ENTER
; }
315 *op
= IR_LT
; return LOOPEV_LEAVE
;
319 /* Record FORL/JFORL or FORI/JFORI. */
320 static LoopEvent
rec_for(jit_State
*J
, const BCIns
*fori
, int isforl
)
322 BCReg ra
= bc_a(*fori
);
324 LoopEvent ev
= for_iter(J
, &op
, ra
, isforl
);
325 TRef
*tr
= &J
->base
[ra
];
328 if (isforl
) { /* Handle FORL/JFORL opcodes. */
331 if (tref_ref(idx
) == J
->scev
.idx
) {
336 if (!idx
) idx
= sloadt(J
, (int32_t)(ra
+FORL_IDX
), IRT_NUM
, 0);
338 stop
= fori_arg(J
, fori
, ra
+FORL_STOP
, t
);
339 step
= fori_arg(J
, fori
, ra
+FORL_STEP
, t
);
341 tr
[FORL_IDX
] = idx
= emitir(IRT(IR_ADD
, t
), idx
, step
);
342 } else { /* Handle FORI/JFORI opcodes. */
345 for (i
= FORL_IDX
; i
<= FORL_STEP
; i
++) {
346 lua_assert(J
->base
[ra
+i
] != 0); /* Assumes the slots are already set. */
347 tr
[i
] = lj_ir_tonum(J
, J
->base
[ra
+i
]);
350 stop
= tr
[FORL_STOP
];
351 if (!tref_isk(tr
[FORL_STEP
])) /* Non-const step: need direction guard. */
352 emitir(IRTG(((op
-IR_LT
)>>1)+IR_LT
, IRT_NUM
),
353 tr
[FORL_STEP
], lj_ir_knum_zero(J
));
357 if (ev
== LOOPEV_LEAVE
) {
358 J
->maxslot
= ra
+FORL_EXT
+1;
362 J
->pc
= fori
+bc_j(*fori
)+1;
366 emitir(IRTG(op
, t
), idx
, stop
);
368 if (ev
== LOOPEV_LEAVE
) {
370 J
->pc
= fori
+bc_j(*fori
)+1;
372 J
->maxslot
= ra
+FORL_EXT
+1;
379 /* Record ITERL/JITERL. */
380 static LoopEvent
rec_iterl(jit_State
*J
, const BCIns iterins
)
382 BCReg ra
= bc_a(iterins
);
383 lua_assert(J
->base
[ra
] != 0);
384 if (!tref_isnil(J
->base
[ra
])) { /* Looping back? */
385 J
->base
[ra
-1] = J
->base
[ra
]; /* Copy result of ITERC to control var. */
386 J
->maxslot
= ra
-1+bc_b(J
->pc
[-1]);
387 J
->pc
+= bc_j(iterins
)+1;
396 /* Record LOOP/JLOOP. Now, that was easy. */
397 static LoopEvent
rec_loop(jit_State
*J
, BCReg ra
)
404 /* Check if a loop repeatedly failed to trace because it didn't loop back. */
405 static int innerloopleft(jit_State
*J
, const BCIns
*pc
)
408 for (i
= 0; i
< PENALTY_SLOTS
; i
++)
409 if (mref(J
->penalty
[i
].pc
, const BCIns
) == pc
) {
410 if ((J
->penalty
[i
].reason
== LJ_TRERR_LLEAVE
||
411 J
->penalty
[i
].reason
== LJ_TRERR_LINNER
) &&
412 J
->penalty
[i
].val
>= 2*PENALTY_MIN
)
419 /* Handle the case when an interpreted loop op is hit. */
420 static void rec_loop_interp(jit_State
*J
, const BCIns
*pc
, LoopEvent ev
)
422 if (J
->parent
== 0) {
423 if (pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0) {
425 if (ev
== LOOPEV_LEAVE
) /* Must loop back to form a root trace. */
426 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
427 rec_stop(J
, J
->cur
.traceno
); /* Root trace forms a loop. */
428 } else if (ev
!= LOOPEV_LEAVE
) { /* Entering inner loop? */
429 /* It's usually better to abort here and wait until the inner loop
430 ** is traced. But if the inner loop repeatedly didn't loop back,
431 ** this indicates a low trip count. In this case try unrolling
432 ** an inner loop even in a root trace. But it's better to be a bit
433 ** more conservative here and only do it for very short loops.
435 if (!innerloopleft(J
, pc
))
436 lj_trace_err(J
, LJ_TRERR_LINNER
); /* Root trace hit an inner loop. */
437 if ((J
->loopref
&& J
->cur
.nins
- J
->loopref
> 8) || --J
->loopunroll
< 0)
438 lj_trace_err(J
, LJ_TRERR_LUNROLL
); /* Limit loop unrolling. */
439 J
->loopref
= J
->cur
.nins
;
441 } else if (ev
!= LOOPEV_LEAVE
) { /* Side trace enters an inner loop. */
442 J
->loopref
= J
->cur
.nins
;
443 if (--J
->loopunroll
< 0)
444 lj_trace_err(J
, LJ_TRERR_LUNROLL
); /* Limit loop unrolling. */
445 } /* Side trace continues across a loop that's left or not entered. */
448 /* Handle the case when an already compiled loop op is hit. */
449 static void rec_loop_jit(jit_State
*J
, TraceNo lnk
, LoopEvent ev
)
451 if (J
->parent
== 0) { /* Root trace hit an inner loop. */
452 /* Better let the inner loop spawn a side trace back here. */
453 lj_trace_err(J
, LJ_TRERR_LINNER
);
454 } else if (ev
!= LOOPEV_LEAVE
) { /* Side trace enters a compiled loop. */
455 J
->instunroll
= 0; /* Cannot continue across a compiled loop op. */
456 if (J
->pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0)
457 lnk
= J
->cur
.traceno
; /* Can form an extra loop. */
458 rec_stop(J
, lnk
); /* Link to the loop. */
459 } /* Side trace continues across a loop that's left or not entered. */
462 /* -- Record calls and returns -------------------------------------------- */
464 /* Record call setup. */
465 static void rec_call_setup(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
468 TValue
*functv
= &J
->L
->base
[func
];
469 TRef trfunc
, *fbase
= &J
->base
[func
];
471 for (i
= 0; i
<= nargs
; i
++)
472 getslot(J
, func
+i
); /* Ensure func and all args have a reference. */
473 if (!tref_isfunc(fbase
[0])) { /* Resolve __call metamethod. */
475 copyTV(J
->L
, &ix
.tabv
, functv
);
476 if (!lj_record_mm_lookup(J
, &ix
, MM_call
) || !tref_isfunc(ix
.mobj
))
477 lj_trace_err(J
, LJ_TRERR_NOMM
);
478 for (i
= ++nargs
; i
> 0; i
--) /* Shift arguments up. */
479 fbase
[i
] = fbase
[i
-1];
480 fbase
[0] = ix
.mobj
; /* Replace function. */
484 /* Specialize to the runtime value of the called function. */
485 trfunc
= lj_ir_kfunc(J
, funcV(functv
));
486 emitir(IRTG(IR_EQ
, IRT_FUNC
), fbase
[0], trfunc
);
487 fbase
[0] = trfunc
| TREF_FRAME
;
488 J
->maxslot
= (BCReg
)nargs
;
492 void lj_record_call(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
494 rec_call_setup(J
, func
, nargs
);
498 J
->baseslot
+= func
+1;
501 /* Record tail call. */
502 void lj_record_tailcall(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
504 rec_call_setup(J
, func
, nargs
);
505 if (frame_isvarg(J
->L
->base
- 1)) {
506 BCReg cbase
= (BCReg
)frame_delta(J
->L
->base
- 1);
507 if (--J
->framedepth
< 0)
508 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
509 J
->baseslot
-= (BCReg
)cbase
;
513 /* Move func + args down. */
514 memmove(&J
->base
[-1], &J
->base
[func
], sizeof(TRef
)*(J
->maxslot
+1));
515 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
516 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
517 if (++J
->tailcalled
> J
->loopunroll
)
518 lj_trace_err(J
, LJ_TRERR_LUNROLL
);
521 /* Check unroll limits for down-recursion. */
522 static int check_downrec_unroll(jit_State
*J
, GCproto
*pt
)
525 for (ptref
= J
->chain
[IR_KGC
]; ptref
; ptref
= IR(ptref
)->prev
)
526 if (ir_kgc(IR(ptref
)) == obj2gco(pt
)) {
529 for (ref
= J
->chain
[IR_RETF
]; ref
; ref
= IR(ref
)->prev
)
530 if (IR(ref
)->op1
== ptref
)
533 if (J
->pc
== J
->startpc
) {
534 if (count
+ J
->tailcalled
> J
->param
[JIT_P_recunroll
])
537 lj_trace_err(J
, LJ_TRERR_DOWNREC
);
545 void lj_record_ret(jit_State
*J
, BCReg rbase
, ptrdiff_t gotresults
)
547 TValue
*frame
= J
->L
->base
- 1;
549 for (i
= 0; i
< gotresults
; i
++)
550 getslot(J
, rbase
+i
); /* Ensure all results have a reference. */
551 while (frame_ispcall(frame
)) { /* Immediately resolve pcall() returns. */
552 BCReg cbase
= (BCReg
)frame_delta(frame
);
553 if (--J
->framedepth
< 0)
554 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
555 lua_assert(J
->baseslot
> 1);
558 J
->baseslot
-= (BCReg
)cbase
;
560 J
->base
[--rbase
] = TREF_TRUE
; /* Prepend true to results. */
561 frame
= frame_prevd(frame
);
563 if (frame_isvarg(frame
)) {
564 BCReg cbase
= (BCReg
)frame_delta(frame
);
565 if (--J
->framedepth
< 0) /* NYI: return of vararg func to lower frame. */
566 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
567 lua_assert(J
->baseslot
> 1);
569 J
->baseslot
-= (BCReg
)cbase
;
571 frame
= frame_prevd(frame
);
573 if (frame_islua(frame
)) { /* Return to Lua frame. */
574 BCIns callins
= *(frame_pc(frame
)-1);
575 ptrdiff_t nresults
= bc_b(callins
) ? (ptrdiff_t)bc_b(callins
)-1 :gotresults
;
576 BCReg cbase
= bc_a(callins
);
577 GCproto
*pt
= funcproto(frame_func(frame
- (cbase
+1)));
578 if (J
->framedepth
== 0 && J
->pt
&& frame
== J
->L
->base
- 1) {
579 if (check_downrec_unroll(J
, pt
)) {
580 J
->maxslot
= (BCReg
)(rbase
+ nresults
);
581 rec_stop(J
, J
->cur
.traceno
); /* Down-recursion. */
586 for (i
= 0; i
< nresults
; i
++) /* Adjust results. */
587 J
->base
[i
-1] = i
< gotresults
? J
->base
[rbase
+i
] : TREF_NIL
;
588 J
->maxslot
= cbase
+(BCReg
)nresults
;
589 if (J
->framedepth
> 0) { /* Return to a frame that is part of the trace. */
591 lua_assert(J
->baseslot
> cbase
+1);
592 J
->baseslot
-= cbase
+1;
594 } else if (J
->parent
== 0 && !bc_isret(bc_op(J
->cur
.startins
))) {
595 /* Return to lower frame would leave the loop in a root trace. */
596 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
597 } else { /* Return to lower frame. Guard for the target we return to. */
598 TRef trpt
= lj_ir_kgc(J
, obj2gco(pt
), IRT_PROTO
);
599 TRef trpc
= lj_ir_kptr(J
, (void *)frame_pc(frame
));
600 emitir(IRTG(IR_RETF
, IRT_P32
), trpt
, trpc
);
603 lua_assert(J
->baseslot
== 1);
604 /* Shift result slots up and clear the slots of the new frame below. */
605 memmove(J
->base
+ cbase
, J
->base
-1, sizeof(TRef
)*nresults
);
606 memset(J
->base
-1, 0, sizeof(TRef
)*(cbase
+1));
608 } else if (frame_iscont(frame
)) { /* Return to continuation frame. */
609 ASMFunction cont
= frame_contf(frame
);
610 BCReg cbase
= (BCReg
)frame_delta(frame
);
611 if ((J
->framedepth
-= 2) < 0)
612 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
613 J
->baseslot
-= (BCReg
)cbase
;
615 J
->maxslot
= cbase
-2;
616 if (cont
== lj_cont_ra
) {
617 /* Copy result to destination slot. */
618 BCReg dst
= bc_a(*(frame_contpc(frame
)-1));
619 J
->base
[dst
] = gotresults
? J
->base
[cbase
+rbase
] : TREF_NIL
;
620 if (dst
>= J
->maxslot
) J
->maxslot
= dst
+1;
621 } else if (cont
== lj_cont_nop
) {
622 /* Nothing to do here. */
623 } else if (cont
== lj_cont_cat
) {
626 /* Result type already specialized. */
627 lua_assert(cont
== lj_cont_condf
|| cont
== lj_cont_condt
);
630 lj_trace_err(J
, LJ_TRERR_NYIRETL
); /* NYI: handle return to C frame. */
632 lua_assert(J
->baseslot
>= 1);
635 /* -- Metamethod handling ------------------------------------------------- */
637 /* Prepare to record call to metamethod. */
638 static BCReg
rec_mm_prep(jit_State
*J
, ASMFunction cont
)
640 BCReg s
, top
= curr_proto(J
->L
)->framesize
;
642 setcont(&J
->L
->base
[top
], cont
);
644 trcont
= lj_ir_kptr(J
, (void *)((int64_t)cont
- (int64_t)lj_vm_asm_begin
));
646 trcont
= lj_ir_kptr(J
, (void *)cont
);
648 J
->base
[top
] = trcont
| TREF_CONT
;
650 for (s
= J
->maxslot
; s
< top
; s
++)
651 J
->base
[s
] = 0; /* Clear frame gap to avoid resurrecting previous refs. */
655 /* Record metamethod lookup. */
656 int lj_record_mm_lookup(jit_State
*J
, RecordIndex
*ix
, MMS mm
)
660 if (tref_istab(ix
->tab
)) {
661 mt
= tabref(tabV(&ix
->tabv
)->metatable
);
662 mix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_TAB_META
);
663 } else if (tref_isudata(ix
->tab
)) {
664 int udtype
= udataV(&ix
->tabv
)->udtype
;
665 mt
= tabref(udataV(&ix
->tabv
)->metatable
);
666 /* The metatables of special userdata objects are treated as immutable. */
667 if (udtype
!= UDTYPE_USERDATA
) {
669 if (LJ_HASFFI
&& udtype
== UDTYPE_FFI_CLIB
) {
670 /* Specialize to the C library namespace object. */
671 emitir(IRTG(IR_EQ
, IRT_P32
), ix
->tab
, lj_ir_kptr(J
, udataV(&ix
->tabv
)));
673 /* Specialize to the type of userdata. */
674 TRef tr
= emitir(IRT(IR_FLOAD
, IRT_U8
), ix
->tab
, IRFL_UDATA_UDTYPE
);
675 emitir(IRTGI(IR_EQ
), tr
, lj_ir_kint(J
, udtype
));
678 mo
= lj_tab_getstr(mt
, mmname_str(J2G(J
), mm
));
679 if (!mo
|| tvisnil(mo
))
680 return 0; /* No metamethod. */
681 /* Treat metamethod or index table as immutable, too. */
682 if (!(tvisfunc(mo
) || tvistab(mo
)))
683 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
684 copyTV(J
->L
, &ix
->mobjv
, mo
);
685 ix
->mobj
= lj_ir_kgc(J
, gcV(mo
), tvisfunc(mo
) ? IRT_FUNC
: IRT_TAB
);
687 ix
->mt
= TREF_NIL
; /* Dummy value for comparison semantics. */
688 return 1; /* Got metamethod or index table. */
690 mix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_UDATA_META
);
692 /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
693 mt
= tabref(basemt_obj(J2G(J
), &ix
->tabv
));
696 return 0; /* No metamethod. */
698 /* The cdata metatable is treated as immutable. */
699 if (LJ_HASFFI
&& tref_iscdata(ix
->tab
)) goto immutable_mt
;
700 ix
->mt
= mix
.tab
= lj_ir_ktab(J
, mt
);
703 ix
->mt
= mt
? mix
.tab
: TREF_NIL
;
704 emitir(IRTG(mt
? IR_NE
: IR_EQ
, IRT_TAB
), mix
.tab
, lj_ir_knull(J
, IRT_TAB
));
707 GCstr
*mmstr
= mmname_str(J2G(J
), mm
);
708 cTValue
*mo
= lj_tab_getstr(mt
, mmstr
);
709 if (mo
&& !tvisnil(mo
))
710 copyTV(J
->L
, &ix
->mobjv
, mo
);
712 settabV(J
->L
, &mix
.tabv
, mt
);
713 setstrV(J
->L
, &mix
.keyv
, mmstr
);
714 mix
.key
= lj_ir_kstr(J
, mmstr
);
717 ix
->mobj
= lj_record_idx(J
, &mix
);
718 return !tref_isnil(ix
->mobj
); /* 1 if metamethod found, 0 if not. */
720 return 0; /* No metamethod. */
723 /* Record call to arithmetic metamethod (and MM_len). */
724 static TRef
rec_mm_arith(jit_State
*J
, RecordIndex
*ix
, MMS mm
)
726 /* Set up metamethod call first to save ix->tab and ix->tabv. */
727 BCReg func
= rec_mm_prep(J
, lj_cont_ra
);
728 TRef
*base
= J
->base
+ func
;
729 TValue
*basev
= J
->L
->base
+ func
;
730 base
[1] = ix
->tab
; base
[2] = ix
->key
;
731 copyTV(J
->L
, basev
+1, &ix
->tabv
);
732 copyTV(J
->L
, basev
+2, &ix
->keyv
);
733 if (!lj_record_mm_lookup(J
, ix
, mm
)) { /* Lookup mm on 1st operand. */
736 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
737 if (lj_record_mm_lookup(J
, ix
, mm
)) /* Lookup mm on 2nd operand. */
740 lj_trace_err(J
, LJ_TRERR_NOMM
);
744 copyTV(J
->L
, basev
+0, &ix
->mobjv
);
745 lj_record_call(J
, func
, 2);
746 return 0; /* No result yet. */
749 /* Call a comparison metamethod. */
750 static void rec_mm_callcomp(jit_State
*J
, RecordIndex
*ix
, int op
)
752 BCReg func
= rec_mm_prep(J
, (op
&1) ? lj_cont_condf
: lj_cont_condt
);
753 TRef
*base
= J
->base
+ func
;
754 TValue
*tv
= J
->L
->base
+ func
;
755 base
[0] = ix
->mobj
; base
[1] = ix
->val
; base
[2] = ix
->key
;
756 copyTV(J
->L
, tv
+0, &ix
->mobjv
);
757 copyTV(J
->L
, tv
+1, &ix
->valv
);
758 copyTV(J
->L
, tv
+2, &ix
->keyv
);
759 lj_record_call(J
, func
, 2);
762 /* Record call to equality comparison metamethod (for tab and udata only). */
763 static void rec_mm_equal(jit_State
*J
, RecordIndex
*ix
, int op
)
766 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
767 if (lj_record_mm_lookup(J
, ix
, MM_eq
)) { /* Lookup mm on 1st operand. */
771 copyTV(J
->L
, &mo1v
, &ix
->mobjv
);
772 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
774 if (tvistab(bv
) && tabref(tabV(bv
)->metatable
) == ix
->mtv
) {
775 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_TAB_META
);
776 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
777 } else if (tvisudata(bv
) && tabref(udataV(bv
)->metatable
) == ix
->mtv
) {
778 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_UDATA_META
);
779 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
780 } else { /* Lookup metamethod on 2nd operand and compare both. */
782 copyTV(J
->L
, &ix
->tabv
, bv
);
783 if (!lj_record_mm_lookup(J
, ix
, MM_eq
) ||
784 lj_record_objcmp(J
, mo1
, ix
->mobj
, &mo1v
, &ix
->mobjv
))
787 rec_mm_callcomp(J
, ix
, op
);
791 /* Record call to ordered comparison metamethods (for arbitrary objects). */
792 static void rec_mm_comp(jit_State
*J
, RecordIndex
*ix
, int op
)
795 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
797 MMS mm
= (op
& 2) ? MM_le
: MM_lt
; /* Try __le + __lt or only __lt. */
798 if (lj_record_mm_lookup(J
, ix
, mm
)) { /* Lookup mm on 1st operand. */
802 copyTV(J
->L
, &mo1v
, &ix
->mobjv
);
803 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
805 if (tvistab(bv
) && tabref(tabV(bv
)->metatable
) == ix
->mtv
) {
806 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_TAB_META
);
807 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
808 } else if (tvisudata(bv
) && tabref(udataV(bv
)->metatable
) == ix
->mtv
) {
809 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_UDATA_META
);
810 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
811 } else { /* Lookup metamethod on 2nd operand and compare both. */
813 copyTV(J
->L
, &ix
->tabv
, bv
);
814 if (!lj_record_mm_lookup(J
, ix
, mm
) ||
815 lj_record_objcmp(J
, mo1
, ix
->mobj
, &mo1v
, &ix
->mobjv
))
818 rec_mm_callcomp(J
, ix
, op
);
822 /* First lookup failed. Retry with __lt and swapped operands. */
823 if (!(op
& 2)) break; /* Already at __lt. Interpreter will throw. */
824 ix
->tab
= ix
->key
; ix
->key
= ix
->val
; ix
->val
= ix
->tab
;
825 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
826 copyTV(J
->L
, &ix
->keyv
, &ix
->valv
);
827 copyTV(J
->L
, &ix
->valv
, &ix
->tabv
);
833 /* Setup call to cdata comparison metamethod. */
834 static void rec_mm_comp_cdata(jit_State
*J
, RecordIndex
*ix
, int op
, MMS mm
)
837 if (tref_iscdata(ix
->val
)) {
839 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
841 lua_assert(tref_iscdata(ix
->key
));
843 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
845 lj_record_mm_lookup(J
, ix
, mm
);
846 rec_mm_callcomp(J
, ix
, op
);
850 /* -- Indexed access ------------------------------------------------------ */
852 /* Record bounds-check. */
853 static void rec_idx_abc(jit_State
*J
, TRef asizeref
, TRef ikey
, uint32_t asize
)
855 /* Try to emit invariant bounds checks. */
856 if ((J
->flags
& (JIT_F_OPT_LOOP
|JIT_F_OPT_ABC
)) ==
857 (JIT_F_OPT_LOOP
|JIT_F_OPT_ABC
)) {
858 IRRef ref
= tref_ref(ikey
);
862 /* Handle constant offsets. */
863 if (ir
->o
== IR_ADD
&& irref_isk(ir
->op2
)) {
869 /* Got scalar evolution analysis results for this reference? */
870 if (ref
== J
->scev
.idx
) {
872 lua_assert(irt_isint(J
->scev
.t
) && ir
->o
== IR_SLOAD
);
873 stop
= lj_num2int(numV(&(J
->L
->base
- J
->baseslot
)[ir
->op1
+ FORL_STOP
]));
874 /* Runtime value for stop of loop is within bounds? */
875 if ((int64_t)stop
+ ofs
< (int64_t)asize
) {
876 /* Emit invariant bounds check for stop. */
877 emitir(IRTG(IR_ABC
, IRT_P32
), asizeref
, ofs
== 0 ? J
->scev
.stop
:
878 emitir(IRTI(IR_ADD
), J
->scev
.stop
, ofsref
));
879 /* Emit invariant bounds check for start, if not const or negative. */
880 if (!(J
->scev
.dir
&& J
->scev
.start
&&
881 (int64_t)IR(J
->scev
.start
)->i
+ ofs
>= 0))
882 emitir(IRTG(IR_ABC
, IRT_P32
), asizeref
, ikey
);
887 emitir(IRTGI(IR_ABC
), asizeref
, ikey
); /* Emit regular bounds check. */
890 /* Record indexed key lookup. */
891 static TRef
rec_idx_key(jit_State
*J
, RecordIndex
*ix
)
894 GCtab
*t
= tabV(&ix
->tabv
);
895 ix
->oldv
= lj_tab_get(J
->L
, t
, &ix
->keyv
); /* Lookup previous value. */
897 /* Integer keys are looked up in the array part first. */
899 if (tref_isnumber(key
)) {
900 lua_Number n
= numV(&ix
->keyv
);
901 int32_t k
= lj_num2int(n
);
902 lua_assert(tvisnum(&ix
->keyv
));
903 /* Potential array key? */
904 if ((MSize
)k
< LJ_MAX_ASIZE
&& n
== cast_num(k
)) {
905 TRef asizeref
, ikey
= key
;
906 if (!tref_isinteger(ikey
))
907 ikey
= emitir(IRTGI(IR_CONV
), ikey
, IRCONV_INT_NUM
|IRCONV_INDEX
);
908 asizeref
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_ASIZE
);
909 if ((MSize
)k
< t
->asize
) { /* Currently an array key? */
911 rec_idx_abc(J
, asizeref
, ikey
, t
->asize
);
912 arrayref
= emitir(IRT(IR_FLOAD
, IRT_P32
), ix
->tab
, IRFL_TAB_ARRAY
);
913 return emitir(IRT(IR_AREF
, IRT_P32
), arrayref
, ikey
);
914 } else { /* Currently not in array (may be an array extension)? */
915 emitir(IRTGI(IR_ULE
), asizeref
, ikey
); /* Inv. bounds check. */
916 if (k
== 0 && tref_isk(key
))
917 key
= lj_ir_knum_zero(J
); /* Canonicalize 0 or +-0.0 to +0.0. */
918 /* And continue with the hash lookup. */
920 } else if (!tref_isk(key
)) {
921 /* We can rule out const numbers which failed the integerness test
922 ** above. But all other numbers are potential array keys.
924 if (t
->asize
== 0) { /* True sparse tables have an empty array part. */
925 /* Guard that the array part stays empty. */
926 TRef tmp
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_ASIZE
);
927 emitir(IRTGI(IR_EQ
), tmp
, lj_ir_kint(J
, 0));
929 lj_trace_err(J
, LJ_TRERR_NYITMIX
);
934 /* Otherwise the key is located in the hash part. */
935 if (tref_isinteger(key
)) /* Hash keys are based on numbers, not ints. */
936 ix
->key
= key
= emitir(IRTN(IR_CONV
), key
, IRCONV_NUM_INT
);
938 /* Optimize lookup of constant hash keys. */
939 MSize hslot
= (MSize
)((char *)ix
->oldv
- (char *)&noderef(t
->node
)[0].val
);
940 if (t
->hmask
> 0 && hslot
<= t
->hmask
*(MSize
)sizeof(Node
) &&
941 hslot
<= 65535*(MSize
)sizeof(Node
)) {
943 TRef hm
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_HMASK
);
944 emitir(IRTGI(IR_EQ
), hm
, lj_ir_kint(J
, (int32_t)t
->hmask
));
945 node
= emitir(IRT(IR_FLOAD
, IRT_P32
), ix
->tab
, IRFL_TAB_NODE
);
946 kslot
= lj_ir_kslot(J
, key
, hslot
/ sizeof(Node
));
947 return emitir(IRTG(IR_HREFK
, IRT_P32
), node
, kslot
);
950 /* Fall back to a regular hash lookup. */
951 return emitir(IRT(IR_HREF
, IRT_P32
), ix
->tab
, key
);
954 /* Determine whether a key is NOT one of the fast metamethod names. */
955 static int nommstr(jit_State
*J
, TRef key
)
957 if (tref_isstr(key
)) {
959 GCstr
*str
= ir_kstr(IR(tref_ref(key
)));
961 for (mm
= 0; mm
<= MM_FAST
; mm
++)
962 if (mmname_str(J2G(J
), mm
) == str
)
963 return 0; /* MUST be one the fast metamethod names. */
965 return 0; /* Variable string key MAY be a metamethod name. */
968 return 1; /* CANNOT be a metamethod name. */
971 /* Record indexed load/store. */
972 TRef
lj_record_idx(jit_State
*J
, RecordIndex
*ix
)
978 while (!tref_istab(ix
->tab
)) { /* Handle non-table lookup. */
979 /* Never call raw lj_record_idx() on non-table. */
980 lua_assert(ix
->idxchain
!= 0);
981 if (!lj_record_mm_lookup(J
, ix
, ix
->val
? MM_newindex
: MM_index
))
982 lj_trace_err(J
, LJ_TRERR_NOMM
);
984 if (tref_isfunc(ix
->mobj
)) { /* Handle metamethod call. */
985 BCReg func
= rec_mm_prep(J
, ix
->val
? lj_cont_nop
: lj_cont_ra
);
986 TRef
*base
= J
->base
+ func
;
987 TValue
*tv
= J
->L
->base
+ func
;
988 base
[0] = ix
->mobj
; base
[1] = ix
->tab
; base
[2] = ix
->key
;
989 setfuncV(J
->L
, tv
+0, funcV(&ix
->mobjv
));
990 copyTV(J
->L
, tv
+1, &ix
->tabv
);
991 copyTV(J
->L
, tv
+2, &ix
->keyv
);
994 copyTV(J
->L
, tv
+3, &ix
->valv
);
995 lj_record_call(J
, func
, 3); /* mobj(tab, key, val) */
998 lj_record_call(J
, func
, 2); /* res = mobj(tab, key) */
999 return 0; /* No result yet. */
1002 /* Otherwise retry lookup with metaobject. */
1004 copyTV(J
->L
, &ix
->tabv
, &ix
->mobjv
);
1005 if (--ix
->idxchain
== 0)
1006 lj_trace_err(J
, LJ_TRERR_IDXLOOP
);
1009 /* First catch nil and NaN keys for tables. */
1010 if (tvisnil(&ix
->keyv
) || (tvisnum(&ix
->keyv
) && tvisnan(&ix
->keyv
))) {
1011 if (ix
->val
) /* Better fail early. */
1012 lj_trace_err(J
, LJ_TRERR_STORENN
);
1013 if (tref_isk(ix
->key
)) {
1014 if (ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_index
))
1020 /* Record the key lookup. */
1021 xref
= rec_idx_key(J
, ix
);
1022 xrefop
= IR(tref_ref(xref
))->o
;
1023 loadop
= xrefop
== IR_AREF
? IR_ALOAD
: IR_HLOAD
;
1024 /* NYI: workaround until lj_meta_tset() inconsistency is solved. */
1025 oldv
= xrefop
== IR_KKPTR
? (cTValue
*)ir_kptr(IR(tref_ref(xref
))) : ix
->oldv
;
1027 if (ix
->val
== 0) { /* Indexed load */
1028 IRType t
= itype2irt(oldv
);
1030 if (oldv
== niltvg(J2G(J
))) {
1031 emitir(IRTG(IR_EQ
, IRT_P32
), xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1034 res
= emitir(IRTG(loadop
, t
), xref
, 0);
1036 if (t
== IRT_NIL
&& ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_index
))
1038 if (irtype_ispri(t
)) res
= TREF_PRI(t
); /* Canonicalize primitives. */
1040 } else { /* Indexed store. */
1041 GCtab
*mt
= tabref(tabV(&ix
->tabv
)->metatable
);
1042 int keybarrier
= tref_isgcv(ix
->key
) && !tref_isnil(ix
->val
);
1043 if (tvisnil(oldv
)) { /* Previous value was nil? */
1044 /* Need to duplicate the hasmm check for the early guards. */
1046 if (ix
->idxchain
&& mt
) {
1047 cTValue
*mo
= lj_tab_getstr(mt
, mmname_str(J2G(J
), MM_newindex
));
1048 hasmm
= mo
&& !tvisnil(mo
);
1051 emitir(IRTG(loadop
, IRT_NIL
), xref
, 0); /* Guard for nil value. */
1052 else if (xrefop
== IR_HREF
)
1053 emitir(IRTG(oldv
== niltvg(J2G(J
)) ? IR_EQ
: IR_NE
, IRT_P32
),
1054 xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1055 if (ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_newindex
)) {
1060 if (oldv
== niltvg(J2G(J
))) { /* Need to insert a new key. */
1062 if (tref_isinteger(key
)) /* NEWREF needs a TValue as a key. */
1063 key
= emitir(IRTN(IR_CONV
), key
, IRCONV_NUM_INT
);
1064 xref
= emitir(IRT(IR_NEWREF
, IRT_P32
), ix
->tab
, key
);
1065 keybarrier
= 0; /* NEWREF already takes care of the key barrier. */
1067 } else if (!lj_opt_fwd_wasnonnil(J
, loadop
, tref_ref(xref
))) {
1068 /* Cannot derive that the previous value was non-nil, must do checks. */
1069 if (xrefop
== IR_HREF
) /* Guard against store to niltv. */
1070 emitir(IRTG(IR_NE
, IRT_P32
), xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1071 if (ix
->idxchain
) { /* Metamethod lookup required? */
1072 /* A check for NULL metatable is cheaper (hoistable) than a load. */
1074 TRef mtref
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_TAB_META
);
1075 emitir(IRTG(IR_EQ
, IRT_TAB
), mtref
, lj_ir_knull(J
, IRT_TAB
));
1077 IRType t
= itype2irt(oldv
);
1078 emitir(IRTG(loadop
, t
), xref
, 0); /* Guard for non-nil value. */
1082 keybarrier
= 0; /* Previous non-nil value kept the key alive. */
1084 if (tref_isinteger(ix
->val
)) /* Convert int to number before storing. */
1085 ix
->val
= emitir(IRTN(IR_CONV
), ix
->val
, IRCONV_NUM_INT
);
1086 emitir(IRT(loadop
+IRDELTA_L2S
, tref_type(ix
->val
)), xref
, ix
->val
);
1087 if (keybarrier
|| tref_isgcv(ix
->val
))
1088 emitir(IRT(IR_TBAR
, IRT_NIL
), ix
->tab
, 0);
1089 /* Invalidate neg. metamethod cache for stores with certain string keys. */
1090 if (!nommstr(J
, ix
->key
)) {
1091 TRef fref
= emitir(IRT(IR_FREF
, IRT_P32
), ix
->tab
, IRFL_TAB_NOMM
);
1092 emitir(IRT(IR_FSTORE
, IRT_U8
), fref
, lj_ir_kint(J
, 0));
1099 /* -- Upvalue access ------------------------------------------------------ */
1101 /* Record upvalue load/store. */
1102 static TRef
rec_upvalue(jit_State
*J
, uint32_t uv
, TRef val
)
1104 GCupval
*uvp
= &gcref(J
->fn
->l
.uvptr
[uv
])->uv
;
1105 TRef fn
= getcurrf(J
);
1107 int needbarrier
= 0;
1108 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1109 uv
= (uv
<< 8) | (hashrot(uvp
->dhash
, uvp
->dhash
+ HASH_BIAS
) & 0xff);
1111 /* In current stack? */
1112 if (uvval(uvp
) >= tvref(J
->L
->stack
) &&
1113 uvval(uvp
) < tvref(J
->L
->maxstack
)) {
1114 int32_t slot
= (int32_t)(uvval(uvp
) - (J
->L
->base
- J
->baseslot
));
1115 if (slot
>= 0) { /* Aliases an SSA slot? */
1116 slot
-= (int32_t)J
->baseslot
; /* Note: slot number may be negative! */
1117 /* NYI: add IR to guard that it's still aliasing the same slot. */
1119 return getslot(J
, slot
);
1121 J
->base
[slot
] = val
;
1122 if (slot
>= (int32_t)J
->maxslot
) J
->maxslot
= (BCReg
)(slot
+1);
1127 uref
= tref_ref(emitir(IRTG(IR_UREFO
, IRT_P32
), fn
, uv
));
1130 uref
= tref_ref(emitir(IRTG(IR_UREFC
, IRT_P32
), fn
, uv
));
1132 if (val
== 0) { /* Upvalue load */
1133 IRType t
= itype2irt(uvval(uvp
));
1134 TRef res
= emitir(IRTG(IR_ULOAD
, t
), uref
, 0);
1135 if (irtype_ispri(t
)) res
= TREF_PRI(t
); /* Canonicalize primitive refs. */
1137 } else { /* Upvalue store. */
1138 if (tref_isinteger(val
)) /* Convert int to number before storing. */
1139 val
= emitir(IRTN(IR_CONV
), val
, IRCONV_NUM_INT
);
1140 emitir(IRT(IR_USTORE
, tref_type(val
)), uref
, val
);
1141 if (needbarrier
&& tref_isgcv(val
))
1142 emitir(IRT(IR_OBAR
, IRT_NIL
), uref
, val
);
1148 /* -- Record calls to Lua functions --------------------------------------- */
1150 /* Check unroll limits for calls. */
1151 static void check_call_unroll(jit_State
*J
)
1153 IRRef fref
= tref_ref(J
->base
[-1]);
1156 for (s
= J
->baseslot
- 1; s
> 0; s
--)
1157 if ((J
->slot
[s
] & TREF_FRAME
) && tref_ref(J
->slot
[s
]) == fref
)
1159 if (J
->pc
== J
->startpc
) {
1160 if (count
+ J
->tailcalled
> J
->param
[JIT_P_recunroll
]) {
1162 rec_stop(J
, J
->cur
.traceno
); /* Up-recursion or tail-recursion. */
1165 if (count
> J
->param
[JIT_P_callunroll
])
1166 lj_trace_err(J
, LJ_TRERR_CUNROLL
);
1170 /* Record Lua function setup. */
1171 static void rec_func_setup(jit_State
*J
)
1173 GCproto
*pt
= J
->pt
;
1174 BCReg s
, numparams
= pt
->numparams
;
1175 if ((pt
->flags
& PROTO_NO_JIT
))
1176 lj_trace_err(J
, LJ_TRERR_CJITOFF
);
1177 if (J
->baseslot
+ pt
->framesize
>= LJ_MAX_JSLOTS
)
1178 lj_trace_err(J
, LJ_TRERR_STACKOV
);
1179 /* Fill up missing parameters with nil. */
1180 for (s
= J
->maxslot
; s
< numparams
; s
++)
1181 J
->base
[s
] = TREF_NIL
;
1182 /* The remaining slots should never be read before they are written. */
1183 J
->maxslot
= numparams
;
1186 /* Record Lua vararg function setup. */
1187 static void rec_func_vararg(jit_State
*J
)
1189 GCproto
*pt
= J
->pt
;
1190 BCReg s
, fixargs
, vframe
= J
->maxslot
+1;
1191 lua_assert((pt
->flags
& PROTO_IS_VARARG
));
1192 if (J
->baseslot
+ vframe
+ pt
->framesize
>= LJ_MAX_JSLOTS
)
1193 lj_trace_err(J
, LJ_TRERR_STACKOV
);
1194 J
->base
[vframe
-1] = J
->base
[-1]; /* Copy function up. */
1195 /* Copy fixarg slots up and set their original slots to nil. */
1196 fixargs
= pt
->numparams
< J
->maxslot
? pt
->numparams
: J
->maxslot
;
1197 for (s
= 0; s
< fixargs
; s
++) {
1198 J
->base
[vframe
+s
] = J
->base
[s
];
1199 J
->base
[s
] = TREF_NIL
;
1201 J
->maxslot
= fixargs
;
1204 J
->baseslot
+= vframe
;
1207 /* Record entry to a Lua function. */
1208 static void rec_func_lua(jit_State
*J
)
1211 check_call_unroll(J
);
1214 /* Record entry to an already compiled function. */
1215 static void rec_func_jit(jit_State
*J
, TraceNo lnk
)
1218 J
->instunroll
= 0; /* Cannot continue across a compiled function. */
1219 if (J
->pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0)
1220 lnk
= J
->cur
.traceno
; /* Can form an extra tail-recursive loop. */
1221 rec_stop(J
, lnk
); /* Link to the function. */
1224 /* -- Vararg handling ----------------------------------------------------- */
1226 /* Detect y = select(x, ...) idiom. */
1227 static int select_detect(jit_State
*J
)
1229 BCIns ins
= J
->pc
[1];
1230 if (bc_op(ins
) == BC_CALLM
&& bc_b(ins
) == 2 && bc_c(ins
) == 1) {
1231 cTValue
*func
= &J
->L
->base
[bc_a(ins
)];
1232 if (tvisfunc(func
) && funcV(func
)->c
.ffid
== FF_select
)
1238 /* Record vararg instruction. */
1239 static void rec_varg(jit_State
*J
, BCReg dst
, ptrdiff_t nresults
)
1241 int32_t numparams
= J
->pt
->numparams
;
1242 ptrdiff_t nvararg
= frame_delta(J
->L
->base
-1) - numparams
- 1;
1243 lua_assert(frame_isvarg(J
->L
->base
-1));
1244 if (J
->framedepth
> 0) { /* Simple case: varargs defined on-trace. */
1246 if (nvararg
< 0) nvararg
= 0;
1247 if (nresults
== -1) {
1249 J
->maxslot
= dst
+ (BCReg
)nvararg
;
1250 } else if (dst
+ nresults
> J
->maxslot
) {
1251 J
->maxslot
= dst
+ (BCReg
)nresults
;
1253 for (i
= 0; i
< nresults
; i
++) {
1254 J
->base
[dst
+i
] = i
< nvararg
? J
->base
[i
- nvararg
- 1] : TREF_NIL
;
1255 lua_assert(J
->base
[dst
+i
] != 0);
1257 } else { /* Unknown number of varargs passed to trace. */
1258 TRef fr
= emitir(IRTI(IR_SLOAD
), 0, IRSLOAD_READONLY
|IRSLOAD_FRAME
);
1259 int32_t frofs
= 8*(1+numparams
)+FRAME_VARG
;
1260 if (nresults
>= 0) { /* Known fixed number of results. */
1263 ptrdiff_t nload
= nvararg
>= nresults
? nresults
: nvararg
;
1265 if (nvararg
>= nresults
)
1266 emitir(IRTGI(IR_GE
), fr
, lj_ir_kint(J
, frofs
+8*(int32_t)nresults
));
1268 emitir(IRTGI(IR_EQ
), fr
, lj_ir_kint(J
, frame_ftsz(J
->L
->base
-1)));
1269 vbase
= emitir(IRTI(IR_SUB
), REF_BASE
, fr
);
1270 vbase
= emitir(IRT(IR_ADD
, IRT_P32
), vbase
, lj_ir_kint(J
, frofs
-8));
1271 for (i
= 0; i
< nload
; i
++) {
1272 IRType t
= itype2irt(&J
->L
->base
[i
-1-nvararg
]);
1273 TRef aref
= emitir(IRT(IR_AREF
, IRT_P32
),
1274 vbase
, lj_ir_kint(J
, (int32_t)i
));
1275 TRef tr
= emitir(IRTG(IR_VLOAD
, t
), aref
, 0);
1276 if (irtype_ispri(t
)) tr
= TREF_PRI(t
); /* Canonicalize primitives. */
1277 J
->base
[dst
+i
] = tr
;
1280 emitir(IRTGI(IR_LE
), fr
, lj_ir_kint(J
, frofs
));
1283 for (i
= nvararg
; i
< nresults
; i
++)
1284 J
->base
[dst
+i
] = TREF_NIL
;
1285 if (dst
+ (BCReg
)nresults
> J
->maxslot
)
1286 J
->maxslot
= dst
+ (BCReg
)nresults
;
1287 } else if (select_detect(J
)) { /* y = select(x, ...) */
1288 TRef tridx
= J
->base
[dst
-1];
1290 ptrdiff_t idx
= lj_ffrecord_select_mode(J
, tridx
, &J
->L
->base
[dst
-1]);
1291 if (idx
< 0) goto nyivarg
;
1292 if (idx
!= 0 && !tref_isinteger(tridx
))
1293 tridx
= emitir(IRTGI(IR_CONV
), tridx
, IRCONV_INT_NUM
|IRCONV_INDEX
);
1294 if (idx
!= 0 && tref_isk(tridx
)) {
1295 emitir(IRTGI(idx
<= nvararg
? IR_GE
: IR_LT
),
1296 fr
, lj_ir_kint(J
, frofs
+8*(int32_t)idx
));
1297 frofs
-= 8; /* Bias for 1-based index. */
1298 } else if (idx
<= nvararg
) { /* Compute size. */
1299 TRef tmp
= emitir(IRTI(IR_ADD
), fr
, lj_ir_kint(J
, -frofs
));
1301 emitir(IRTGI(IR_GE
), tmp
, lj_ir_kint(J
, 0));
1302 tr
= emitir(IRTI(IR_BSHR
), tmp
, lj_ir_kint(J
, 3));
1304 tridx
= emitir(IRTI(IR_ADD
), tridx
, lj_ir_kint(J
, -1));
1305 rec_idx_abc(J
, tr
, tridx
, (uint32_t)nvararg
);
1308 TRef tmp
= lj_ir_kint(J
, frofs
);
1310 TRef tmp2
= emitir(IRTI(IR_BSHL
), tridx
, lj_ir_kint(J
, 3));
1311 tmp
= emitir(IRTI(IR_ADD
), tmp2
, tmp
);
1313 tr
= lj_ir_kint(J
, 0);
1315 emitir(IRTGI(IR_LT
), fr
, tmp
);
1317 if (idx
!= 0 && idx
<= nvararg
) {
1319 TRef aref
, vbase
= emitir(IRTI(IR_SUB
), REF_BASE
, fr
);
1320 vbase
= emitir(IRT(IR_ADD
, IRT_P32
), vbase
, lj_ir_kint(J
, frofs
-8));
1321 t
= itype2irt(&J
->L
->base
[idx
-2-nvararg
]);
1322 aref
= emitir(IRT(IR_AREF
, IRT_P32
), vbase
, tridx
);
1323 tr
= emitir(IRTG(IR_VLOAD
, t
), aref
, 0);
1324 if (irtype_ispri(t
)) tr
= TREF_PRI(t
); /* Canonicalize primitives. */
1326 J
->base
[dst
-2] = tr
;
1328 J
->bcskip
= 2; /* Skip CALLM + select. */
1331 setintV(&J
->errinfo
, BC_VARG
);
1332 lj_trace_err_info(J
, LJ_TRERR_NYIBC
);
1337 /* -- Record allocations -------------------------------------------------- */
1339 static TRef
rec_tnew(jit_State
*J
, uint32_t ah
)
1341 uint32_t asize
= ah
& 0x7ff;
1342 uint32_t hbits
= ah
>> 11;
1343 if (asize
== 0x7ff) asize
= 0x801;
1344 return emitir(IRTG(IR_TNEW
, IRT_TAB
), asize
, hbits
);
1347 /* -- Record bytecode ops ------------------------------------------------- */
1349 /* Prepare for comparison. */
1350 static void rec_comp_prep(jit_State
*J
)
1352 /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
1353 if (J
->cur
.nsnap
== 1 && J
->cur
.snap
[0].ref
== J
->cur
.nins
)
1354 emitir_raw(IRT(IR_NOP
, IRT_NIL
), 0, 0);
1358 /* Fixup comparison. */
1359 static void rec_comp_fixup(jit_State
*J
, const BCIns
*pc
, int cond
)
1361 BCIns jmpins
= pc
[1];
1362 const BCIns
*npc
= pc
+ 2 + (cond
? bc_j(jmpins
) : 0);
1363 SnapShot
*snap
= &J
->cur
.snap
[J
->cur
.nsnap
-1];
1364 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
1365 J
->cur
.snapmap
[snap
->mapofs
+ snap
->nent
] = SNAP_MKPC(npc
);
1367 /* Shrink last snapshot if possible. */
1368 if (bc_a(jmpins
) < J
->maxslot
) {
1369 J
->maxslot
= bc_a(jmpins
);
1374 /* Record the next bytecode instruction (_before_ it's executed). */
1375 void lj_record_ins(jit_State
*J
)
1384 /* Perform post-processing action before recording the next instruction. */
1385 if (LJ_UNLIKELY(J
->postproc
!= LJ_POST_NONE
)) {
1386 switch (J
->postproc
) {
1387 case LJ_POST_FIXCOMP
: /* Fixup comparison. */
1388 pc
= frame_pc(&J2G(J
)->tmptv
);
1389 rec_comp_fixup(J
, pc
, (!tvistruecond(&J2G(J
)->tmptv2
) ^ (bc_op(*pc
)&1)));
1391 case LJ_POST_FIXGUARD
: /* Fixup and emit pending guard. */
1392 if (!tvistruecond(&J2G(J
)->tmptv2
))
1393 J
->fold
.ins
.o
^= 1; /* Flip guard to opposite. */
1394 lj_opt_fold(J
); /* Emit pending guard. */
1396 case LJ_POST_FIXBOOL
:
1397 if (!tvistruecond(&J2G(J
)->tmptv2
)) {
1399 for (s
= 0; s
< J
->maxslot
; s
++) /* Fixup stack slot (if any). */
1400 if (J
->base
[s
] == TREF_TRUE
&& tvisfalse(&J
->L
->base
[s
])) {
1401 J
->base
[s
] = TREF_FALSE
;
1406 default: lua_assert(0); break;
1408 J
->postproc
= LJ_POST_NONE
;
1411 /* Need snapshot before recording next bytecode (e.g. after a store). */
1418 /* Skip some bytecodes. */
1419 if (LJ_UNLIKELY(J
->bcskip
> 0)) {
1424 /* Record only closed loops for root traces. */
1426 if (J
->framedepth
== 0 &&
1427 (MSize
)((char *)pc
- (char *)J
->bc_min
) >= J
->bc_extent
)
1428 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
1430 #ifdef LUA_USE_ASSERT
1435 /* Keep a copy of the runtime values of var/num/str operands. */
1436 #define rav (&ix.valv)
1437 #define rbv (&ix.tabv)
1438 #define rcv (&ix.keyv)
1445 switch (bcmode_a(op
)) {
1447 copyTV(J
->L
, rav
, &lbase
[ra
]); ix
.val
= ra
= getslot(J
, ra
); break;
1448 default: break; /* Handled later. */
1452 switch (bcmode_b(op
)) {
1453 case BCMnone
: rb
= 0; rc
= bc_d(ins
); break; /* Upgrade rc to 'rd'. */
1455 copyTV(J
->L
, rbv
, &lbase
[rb
]); ix
.tab
= rb
= getslot(J
, rb
); break;
1456 case BCMnum
: { lua_Number n
= proto_knum(J
->pt
, rb
);
1457 setnumV(rbv
, n
); ix
.tab
= rb
= lj_ir_knumint(J
, n
); } break;
1458 default: break; /* Handled later. */
1460 switch (bcmode_c(op
)) {
1462 copyTV(J
->L
, rcv
, &lbase
[rc
]); ix
.key
= rc
= getslot(J
, rc
); break;
1463 case BCMpri
: setitype(rcv
, ~rc
); ix
.key
= rc
= TREF_PRI(IRT_NIL
+rc
); break;
1464 case BCMnum
: { lua_Number n
= proto_knum(J
->pt
, rc
);
1465 setnumV(rcv
, n
); ix
.key
= rc
= lj_ir_knumint(J
, n
); } break;
1466 case BCMstr
: { GCstr
*s
= gco2str(proto_kgc(J
->pt
, ~(ptrdiff_t)rc
));
1467 setstrV(J
->L
, rcv
, s
); ix
.key
= rc
= lj_ir_kstr(J
, s
); } break;
1468 default: break; /* Handled later. */
1473 /* -- Comparison ops ---------------------------------------------------- */
1475 case BC_ISLT
: case BC_ISGE
: case BC_ISLE
: case BC_ISGT
:
1477 if (tref_iscdata(ra
) || tref_iscdata(rc
)) {
1478 rec_mm_comp_cdata(J
, &ix
, op
, ((int)op
& 2) ? MM_le
: MM_lt
);
1482 /* Emit nothing for two numeric or string consts. */
1483 if (!(tref_isk2(ra
,rc
) && tref_isnumber_str(ra
) && tref_isnumber_str(rc
))) {
1484 IRType ta
= tref_isinteger(ra
) ? IRT_INT
: tref_type(ra
);
1485 IRType tc
= tref_isinteger(rc
) ? IRT_INT
: tref_type(rc
);
1488 /* Widen mixed number/int comparisons to number/number comparison. */
1489 if (ta
== IRT_INT
&& tc
== IRT_NUM
) {
1490 ra
= emitir(IRTN(IR_CONV
), ra
, IRCONV_NUM_INT
);
1492 } else if (ta
== IRT_NUM
&& tc
== IRT_INT
) {
1493 rc
= emitir(IRTN(IR_CONV
), rc
, IRCONV_NUM_INT
);
1494 } else if (!((ta
== IRT_FALSE
|| ta
== IRT_TRUE
) &&
1495 (tc
== IRT_FALSE
|| tc
== IRT_TRUE
))) {
1496 break; /* Interpreter will throw for two different types. */
1500 irop
= (int)op
- (int)BC_ISLT
+ (int)IR_LT
;
1501 if (ta
== IRT_NUM
) {
1502 if ((irop
& 1)) irop
^= 4; /* ISGE/ISGT are unordered. */
1503 if (!lj_ir_numcmp(numV(rav
), numV(rcv
), (IROp
)irop
)) irop
^= 5;
1504 } else if (ta
== IRT_INT
) {
1505 if (!lj_ir_numcmp(numV(rav
), numV(rcv
), (IROp
)irop
)) irop
^= 1;
1506 } else if (ta
== IRT_STR
) {
1507 if (!lj_ir_strcmp(strV(rav
), strV(rcv
), (IROp
)irop
)) irop
^= 1;
1508 ra
= lj_ir_call(J
, IRCALL_lj_str_cmp
, ra
, rc
);
1509 rc
= lj_ir_kint(J
, 0);
1512 rec_mm_comp(J
, &ix
, (int)op
);
1515 emitir(IRTG(irop
, ta
), ra
, rc
);
1516 rec_comp_fixup(J
, J
->pc
, ((int)op
^ irop
) & 1);
1520 case BC_ISEQV
: case BC_ISNEV
:
1521 case BC_ISEQS
: case BC_ISNES
:
1522 case BC_ISEQN
: case BC_ISNEN
:
1523 case BC_ISEQP
: case BC_ISNEP
:
1525 if (tref_iscdata(ra
) || tref_iscdata(rc
)) {
1526 rec_mm_comp_cdata(J
, &ix
, op
, MM_eq
);
1530 /* Emit nothing for two non-table, non-udata consts. */
1531 if (!(tref_isk2(ra
, rc
) && !(tref_istab(ra
) || tref_isudata(ra
)))) {
1534 diff
= lj_record_objcmp(J
, ra
, rc
, rav
, rcv
);
1535 if (diff
== 1 && (tref_istab(ra
) || tref_isudata(ra
))) {
1536 /* Only check __eq if different, but the same type (table or udata). */
1537 rec_mm_equal(J
, &ix
, (int)op
);
1540 rec_comp_fixup(J
, J
->pc
, ((int)op
& 1) == !diff
);
1544 /* -- Unary test and copy ops ------------------------------------------- */
1546 case BC_ISTC
: case BC_ISFC
:
1547 if ((op
& 1) == tref_istruecond(rc
))
1548 rc
= 0; /* Don't store if condition is not true. */
1550 case BC_IST
: case BC_ISF
: /* Type specialization suffices. */
1551 if (bc_a(pc
[1]) < J
->maxslot
)
1552 J
->maxslot
= bc_a(pc
[1]); /* Shrink used slots. */
1555 /* -- Unary ops --------------------------------------------------------- */
1558 /* Type specialization already forces const result. */
1559 rc
= tref_istruecond(rc
) ? TREF_FALSE
: TREF_TRUE
;
1563 if (tref_isstr(rc
)) {
1564 rc
= emitir(IRTI(IR_FLOAD
), rc
, IRFL_STR_LEN
);
1565 } else if (tref_istab(rc
)) {
1566 rc
= lj_ir_call(J
, IRCALL_lj_tab_len
, rc
);
1569 copyTV(J
->L
, &ix
.tabv
, &ix
.keyv
);
1572 rc
= rec_mm_arith(J
, &ix
, MM_len
);
1576 /* -- Arithmetic ops ---------------------------------------------------- */
1579 if (tref_isnumber_str(rc
)) {
1580 rc
= lj_ir_tonum(J
, rc
);
1581 rc
= emitir(IRTN(IR_NEG
), rc
, lj_ir_knum_neg(J
));
1584 copyTV(J
->L
, &ix
.tabv
, &ix
.keyv
);
1585 rc
= rec_mm_arith(J
, &ix
, MM_unm
);
1589 case BC_ADDNV
: case BC_SUBNV
: case BC_MULNV
: case BC_DIVNV
: case BC_MODNV
:
1590 ix
.tab
= rc
; ix
.key
= rc
= rb
; rb
= ix
.tab
;
1591 copyTV(J
->L
, &ix
.valv
, &ix
.tabv
);
1592 copyTV(J
->L
, &ix
.tabv
, &ix
.keyv
);
1593 copyTV(J
->L
, &ix
.keyv
, &ix
.valv
);
1597 case BC_ADDVN
: case BC_SUBVN
: case BC_MULVN
: case BC_DIVVN
:
1598 case BC_ADDVV
: case BC_SUBVV
: case BC_MULVV
: case BC_DIVVV
: {
1599 MMS mm
= bcmode_mm(op
);
1600 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
)) {
1601 rb
= lj_ir_tonum(J
, rb
);
1602 rc
= lj_ir_tonum(J
, rc
);
1603 rc
= emitir(IRTN((int)mm
- (int)MM_add
+ (int)IR_ADD
), rb
, rc
);
1605 rc
= rec_mm_arith(J
, &ix
, mm
);
1610 case BC_MODVN
: case BC_MODVV
:
1612 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
))
1613 rc
= lj_opt_narrow_mod(J
, rb
, rc
);
1615 rc
= rec_mm_arith(J
, &ix
, MM_mod
);
1619 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
))
1620 rc
= lj_opt_narrow_pow(J
, lj_ir_tonum(J
, rb
), rc
, rcv
);
1622 rc
= rec_mm_arith(J
, &ix
, MM_pow
);
1625 /* -- Constant and move ops --------------------------------------------- */
1628 /* Clear gap of method call to avoid resurrecting previous refs. */
1629 if (ra
> J
->maxslot
) J
->base
[ra
-1] = 0;
1631 case BC_KSTR
: case BC_KNUM
: case BC_KPRI
:
1634 rc
= lj_ir_kint(J
, (int32_t)(int16_t)rc
);
1638 J
->base
[ra
++] = TREF_NIL
;
1639 if (rc
>= J
->maxslot
) J
->maxslot
= rc
+1;
1643 rc
= lj_ir_kgc(J
, proto_kgc(J
->pt
, ~(ptrdiff_t)rc
), IRT_CDATA
);
1647 /* -- Upvalue and function ops ------------------------------------------ */
1650 rc
= rec_upvalue(J
, rc
, 0);
1652 case BC_USETV
: case BC_USETS
: case BC_USETN
: case BC_USETP
:
1653 rec_upvalue(J
, ra
, rc
);
1656 /* -- Table ops --------------------------------------------------------- */
1658 case BC_GGET
: case BC_GSET
:
1659 settabV(J
->L
, &ix
.tabv
, tabref(J
->fn
->l
.env
));
1660 ix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), getcurrf(J
), IRFL_FUNC_ENV
);
1661 ix
.idxchain
= LJ_MAX_IDXCHAIN
;
1662 rc
= lj_record_idx(J
, &ix
);
1665 case BC_TGETB
: case BC_TSETB
:
1666 setintV(&ix
.keyv
, (int32_t)rc
);
1667 ix
.key
= lj_ir_kint(J
, (int32_t)rc
);
1669 case BC_TGETV
: case BC_TGETS
: case BC_TSETV
: case BC_TSETS
:
1670 ix
.idxchain
= LJ_MAX_IDXCHAIN
;
1671 rc
= lj_record_idx(J
, &ix
);
1675 rc
= rec_tnew(J
, rc
);
1678 rc
= emitir(IRTG(IR_TDUP
, IRT_TAB
),
1679 lj_ir_ktab(J
, gco2tab(proto_kgc(J
->pt
, ~(ptrdiff_t)rc
))), 0);
1682 /* -- Calls and vararg handling ----------------------------------------- */
1685 J
->base
[ra
] = getslot(J
, ra
-3);
1686 J
->base
[ra
+1] = getslot(J
, ra
-2);
1687 J
->base
[ra
+2] = getslot(J
, ra
-1);
1688 { /* Do the actual copy now because lj_record_call needs the values. */
1689 TValue
*b
= &J
->L
->base
[ra
];
1690 copyTV(J
->L
, b
, b
-3);
1691 copyTV(J
->L
, b
+1, b
-2);
1692 copyTV(J
->L
, b
+2, b
-1);
1694 lj_record_call(J
, ra
, (ptrdiff_t)rc
-1);
1697 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
1699 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
;
1702 lj_record_call(J
, ra
, (ptrdiff_t)rc
-1);
1706 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
;
1709 lj_record_tailcall(J
, ra
, (ptrdiff_t)rc
-1);
1713 rec_varg(J
, ra
, (ptrdiff_t)rb
-1);
1716 /* -- Returns ----------------------------------------------------------- */
1719 /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
1720 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
+ 1;
1722 case BC_RET
: case BC_RET0
: case BC_RET1
:
1723 lj_record_ret(J
, ra
, (ptrdiff_t)rc
-1);
1726 /* -- Loops and branches ------------------------------------------------ */
1729 if (rec_for(J
, pc
, 0) != LOOPEV_LEAVE
)
1730 J
->loopref
= J
->cur
.nins
;
1733 lua_assert(bc_op(pc
[(ptrdiff_t)rc
-BCBIAS_J
]) == BC_JFORL
);
1734 if (rec_for(J
, pc
, 0) != LOOPEV_LEAVE
) /* Link to existing loop. */
1735 rec_stop(J
, bc_d(pc
[(ptrdiff_t)rc
-BCBIAS_J
]));
1736 /* Continue tracing if the loop is not entered. */
1740 rec_loop_interp(J
, pc
, rec_for(J
, pc
+((ptrdiff_t)rc
-BCBIAS_J
), 1));
1743 rec_loop_interp(J
, pc
, rec_iterl(J
, *pc
));
1746 rec_loop_interp(J
, pc
, rec_loop(J
, ra
));
1750 rec_loop_jit(J
, rc
, rec_for(J
, pc
+bc_j(traceref(J
, rc
)->startins
), 1));
1753 rec_loop_jit(J
, rc
, rec_iterl(J
, traceref(J
, rc
)->startins
));
1756 rec_loop_jit(J
, rc
, rec_loop(J
, ra
));
1764 lj_trace_err(J
, LJ_TRERR_BLACKL
);
1768 if (ra
< J
->maxslot
)
1769 J
->maxslot
= ra
; /* Shrink used slots. */
1772 /* -- Function headers -------------------------------------------------- */
1778 rec_func_jit(J
, rc
);
1786 lua_assert(0); /* Cannot happen. No hotcall counting for varag funcs. */
1791 lj_ffrecord_func(J
);
1795 if (op
>= BC__MAX
) {
1796 lj_ffrecord_func(J
);
1806 setintV(&J
->errinfo
, (int32_t)op
);
1807 lj_trace_err_info(J
, LJ_TRERR_NYIBC
);
1811 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
1812 if (bcmode_a(op
) == BCMdst
&& rc
) {
1814 if (ra
>= J
->maxslot
) J
->maxslot
= ra
+1;
1821 /* Limit the number of recorded IR instructions. */
1822 if (J
->cur
.nins
> REF_FIRST
+(IRRef
)J
->param
[JIT_P_maxrecord
])
1823 lj_trace_err(J
, LJ_TRERR_TRACEOV
);
1826 /* -- Recording setup ----------------------------------------------------- */
1828 /* Setup recording for a FORL loop. */
1829 static void rec_setup_forl(jit_State
*J
, const BCIns
*fori
)
1831 BCReg ra
= bc_a(*fori
);
1832 cTValue
*forbase
= &J
->L
->base
[ra
];
1833 IRType t
= (J
->flags
& JIT_F_OPT_NARROW
) ? lj_opt_narrow_forl(forbase
)
1836 TRef stop
= fori_arg(J
, fori
, ra
+FORL_STOP
, t
);
1837 TRef step
= fori_arg(J
, fori
, ra
+FORL_STEP
, t
);
1838 int dir
= (0 <= numV(&forbase
[FORL_STEP
]));
1839 lua_assert(bc_op(*fori
) == BC_FORI
|| bc_op(*fori
) == BC_JFORI
);
1842 J
->scev
.stop
= tref_ref(stop
);
1843 J
->scev
.step
= tref_ref(step
);
1844 if (!tref_isk(step
)) {
1845 /* Non-constant step: need a guard for the direction. */
1846 TRef zero
= (t
== IRT_INT
) ? lj_ir_kint(J
, 0) : lj_ir_knum_zero(J
);
1847 emitir(IRTG(dir
? IR_GE
: IR_LT
, t
), step
, zero
);
1848 /* Add hoistable overflow checks for a narrowed FORL index. */
1850 if (tref_isk(stop
)) {
1851 /* Constant stop: optimize check away or to a range check for step. */
1852 int32_t k
= IR(tref_ref(stop
))->i
;
1855 emitir(IRTGI(IR_LE
), step
, lj_ir_kint(J
, (int32_t)0x7fffffff-k
));
1858 emitir(IRTGI(IR_GE
), step
, lj_ir_kint(J
, (int32_t)0x80000000-k
));
1861 /* Stop+step variable: need full overflow check (with dead result). */
1862 emitir(IRTGI(IR_ADDOV
), step
, stop
);
1865 } else if (t
== IRT_INT
&& !tref_isk(stop
)) {
1866 /* Constant step: optimize overflow check to a range check for stop. */
1867 int32_t k
= IR(tref_ref(step
))->i
;
1868 k
= (int32_t)(dir
? 0x7fffffff : 0x80000000) - k
;
1869 emitir(IRTGI(dir
? IR_LE
: IR_GE
), stop
, lj_ir_kint(J
, k
));
1871 J
->scev
.start
= tref_ref(find_kinit(J
, fori
, ra
+FORL_IDX
, IRT_INT
));
1872 start
= sloadt(J
, (int32_t)(ra
+FORL_IDX
),
1873 (t
== IRT_INT
&& !J
->scev
.start
) ? (IRT_INT
|IRT_GUARD
) : t
,
1874 t
== IRT_INT
? (IRSLOAD_CONVERT
|IRSLOAD_INHERIT
) : IRSLOAD_INHERIT
);
1875 J
->base
[ra
+FORL_EXT
] = start
;
1876 J
->scev
.idx
= tref_ref(start
);
1877 J
->maxslot
= ra
+FORL_EXT
+1;
1880 /* Setup recording for a root trace started by a hot loop. */
1881 static const BCIns
*rec_setup_root(jit_State
*J
)
1883 /* Determine the next PC and the bytecode range for the loop. */
1884 const BCIns
*pcj
, *pc
= J
->pc
;
1886 BCReg ra
= bc_a(ins
);
1887 switch (bc_op(ins
)) {
1889 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
1894 lua_assert(bc_op(pc
[-1]) == BC_ITERC
);
1895 J
->maxslot
= ra
+ bc_b(pc
[-1]) - 1;
1896 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
1898 lua_assert(bc_op(pc
[-1]) == BC_JMP
);
1902 /* Only check BC range for real loops, but not for "repeat until true". */
1903 pcj
= pc
+ bc_j(ins
);
1905 if (bc_op(ins
) == BC_JMP
&& bc_j(ins
) < 0) {
1906 J
->bc_min
= pcj
+1 + bc_j(ins
);
1907 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
1915 /* No bytecode range check for down-recursive root traces. */
1916 J
->maxslot
= ra
+ bc_d(ins
);
1919 /* No bytecode range check for root traces started by a hot call. */
1920 J
->maxslot
= J
->pt
->numparams
;
1930 /* Setup recording for a side trace. */
1931 static void rec_setup_side(jit_State
*J
, GCtrace
*T
)
1933 SnapShot
*snap
= &T
->snap
[J
->exitno
];
1934 SnapEntry
*map
= &T
->snapmap
[snap
->mapofs
];
1935 MSize n
, nent
= snap
->nent
;
1936 BloomFilter seen
= 0;
1937 /* Emit IR for slots inherited from parent snapshot. */
1938 for (n
= 0; n
< nent
; n
++) {
1939 SnapEntry sn
= map
[n
];
1940 IRRef ref
= snap_ref(sn
);
1941 BCReg s
= snap_slot(sn
);
1942 IRIns
*ir
= &T
->ir
[ref
];
1944 /* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
1945 if (bloomtest(seen
, ref
)) {
1947 for (j
= 0; j
< n
; j
++)
1948 if (snap_ref(map
[j
]) == ref
) {
1949 tr
= J
->slot
[snap_slot(map
[j
])];
1953 bloomset(seen
, ref
);
1954 switch ((IROp
)ir
->o
) {
1955 /* Only have to deal with constants that can occur in stack slots. */
1956 case IR_KPRI
: tr
= TREF_PRI(irt_type(ir
->t
)); break;
1957 case IR_KINT
: tr
= lj_ir_kint(J
, ir
->i
); break;
1958 case IR_KGC
: tr
= lj_ir_kgc(J
, ir_kgc(ir
), irt_t(ir
->t
)); break;
1959 case IR_KNUM
: tr
= lj_ir_k64(J
, IR_KNUM
, ir_knum(ir
)); break;
1960 case IR_KINT64
: tr
= lj_ir_k64(J
, IR_KINT64
, ir_kint64(ir
)); break;
1961 case IR_KPTR
: tr
= lj_ir_kptr(J
, ir_kptr(ir
)); break; /* Continuation. */
1962 /* Inherited SLOADs don't need a guard or type check. */
1964 tr
= emitir_raw(ir
->ot
& ~IRT_GUARD
, s
,
1965 (ir
->op2
&IRSLOAD_READONLY
) | IRSLOAD_INHERIT
|IRSLOAD_PARENT
);
1967 /* Parent refs are already typed and don't need a guard. */
1969 tr
= emitir_raw(IRT(IR_SLOAD
, irt_type(ir
->t
)), s
,
1970 IRSLOAD_INHERIT
|IRSLOAD_PARENT
);
1974 J
->slot
[s
] = tr
| (sn
&(SNAP_CONT
|SNAP_FRAME
)); /* Same as TREF_* flags. */
1975 if ((sn
& SNAP_FRAME
))
1978 J
->base
= J
->slot
+ J
->baseslot
;
1979 J
->maxslot
= snap
->nslots
- J
->baseslot
;
1980 J
->framedepth
= snap
->depth
;
1984 /* Setup for recording a new trace. */
1985 void lj_record_setup(jit_State
*J
)
1989 /* Initialize state related to current trace. */
1990 memset(J
->slot
, 0, sizeof(J
->slot
));
1991 memset(J
->chain
, 0, sizeof(J
->chain
));
1992 memset(J
->bpropcache
, 0, sizeof(J
->bpropcache
));
1993 J
->scev
.idx
= REF_NIL
;
1995 J
->baseslot
= 1; /* Invoking function is at base[-1]. */
1996 J
->base
= J
->slot
+ J
->baseslot
;
2001 J
->instunroll
= J
->param
[JIT_P_instunroll
];
2002 J
->loopunroll
= J
->param
[JIT_P_loopunroll
];
2006 J
->bc_min
= NULL
; /* Means no limit. */
2007 J
->bc_extent
= ~(MSize
)0;
2009 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2010 emitir_raw(IRT(IR_BASE
, IRT_P32
), J
->parent
, J
->exitno
);
2011 for (i
= 0; i
<= 2; i
++) {
2012 IRIns
*ir
= IR(REF_NIL
-i
);
2014 ir
->t
.irt
= (uint8_t)(IRT_NIL
+i
);
2018 J
->cur
.nk
= REF_TRUE
;
2021 setmref(J
->cur
.startpc
, J
->pc
);
2022 if (J
->parent
) { /* Side trace. */
2023 GCtrace
*T
= traceref(J
, J
->parent
);
2024 TraceNo root
= T
->root
? T
->root
: J
->parent
;
2025 J
->cur
.root
= (uint16_t)root
;
2026 J
->cur
.startins
= BCINS_AD(BC_JMP
, 0, 0);
2027 /* Check whether we could at least potentially form an extra loop. */
2028 if (J
->exitno
== 0 && T
->snap
[0].nent
== 0) {
2029 /* We can narrow a FORL for some side traces, too. */
2030 if (J
->pc
> proto_bc(J
->pt
) && bc_op(J
->pc
[-1]) == BC_JFORI
&&
2031 bc_d(J
->pc
[bc_j(J
->pc
[-1])-1]) == root
) {
2033 rec_setup_forl(J
, J
->pc
-1);
2037 J
->startpc
= NULL
; /* Prevent forming an extra loop. */
2039 rec_setup_side(J
, T
);
2041 if (traceref(J
, J
->cur
.root
)->nchild
>= J
->param
[JIT_P_maxside
] ||
2042 T
->snap
[J
->exitno
].count
>= J
->param
[JIT_P_hotexit
] +
2043 J
->param
[JIT_P_tryside
])
2044 rec_stop(J
, TRACE_INTERP
);
2045 } else { /* Root trace. */
2047 J
->cur
.startins
= *J
->pc
;
2048 J
->pc
= rec_setup_root(J
);
2049 /* Note: the loop instruction itself is recorded at the end and not
2050 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2053 if (bc_op(J
->cur
.startins
) == BC_FORL
)
2054 rec_setup_forl(J
, J
->pc
-1);
2055 if (1 + J
->pt
->framesize
>= LJ_MAX_JSLOTS
)
2056 lj_trace_err(J
, LJ_TRERR_STACKOV
);
2058 #ifdef LUAJIT_ENABLE_CHECKHOOK
2059 /* Regularly check for instruction/line hooks from compiled code and
2060 ** exit to the interpreter if the hooks are set.
2062 ** This is a compile-time option and disabled by default, since the
2063 ** hook checks may be quite expensive in tight loops.
2065 ** Note this is only useful if hooks are *not* set most of the time.
2066 ** Use this only if you want to *asynchronously* interrupt the execution.
2068 ** You can set the instruction hook via lua_sethook() with a count of 1
2069 ** from a signal handler or another native thread. Please have a look
2070 ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2073 TRef tr
= emitir(IRT(IR_XLOAD
, IRT_U8
),
2074 lj_ir_kptr(J
, &J2G(J
)->hookmask
), IRXLOAD_VOLATILE
);
2075 tr
= emitir(IRTI(IR_BAND
), tr
, lj_ir_kint(J
, (LUA_MASKLINE
|LUA_MASKCOUNT
)));
2076 emitir(IRTGI(IR_EQ
), tr
, lj_ir_kint(J
, 0));