2 ** Trace recorder (bytecode -> SSA IR).
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
28 #include "lj_ircall.h"
31 #include "lj_record.h"
32 #include "lj_ffrecord.h"
34 #include "lj_dispatch.h"
38 /* Some local macros to save typing. Undef'd at the end. */
39 #define IR(ref) (&J->cur.ir[(ref)])
41 /* Pass IR on to next optimization in chain (FOLD). */
42 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
44 /* Emit raw IR without passing through optimizations. */
45 #define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
47 /* -- Sanity checks ------------------------------------------------------- */
50 /* Sanity check the whole IR -- sloooow. */
51 static void rec_check_ir(jit_State
*J
)
53 IRRef i
, nins
= J
->cur
.nins
, nk
= J
->cur
.nk
;
54 lj_assertJ(nk
<= REF_BIAS
&& nins
>= REF_BIAS
&& nins
< 65536,
55 "inconsistent IR layout");
56 for (i
= nk
; i
< nins
; i
++) {
58 uint32_t mode
= lj_ir_mode
[ir
->o
];
61 const char *err
= NULL
;
62 switch (irm_op1(mode
)) {
64 if (op1
!= 0) err
= "IRMnone op1 used";
67 if (op1
< nk
|| (i
>= REF_BIAS
? op1
>= i
: op1
<= i
))
68 err
= "IRMref op1 out of range";
72 if (i
>= REF_BIAS
) { err
= "constant in IR range"; break; }
73 if (irt_is64(ir
->t
) && ir
->o
!= IR_KNULL
)
77 switch (irm_op2(mode
)) {
79 if (op2
) err
= "IRMnone op2 used";
82 if (op2
< nk
|| (i
>= REF_BIAS
? op2
>= i
: op2
<= i
))
83 err
= "IRMref op2 out of range";
86 case IRMcst
: err
= "IRMcst op2"; break;
88 if (!err
&& ir
->prev
) {
89 if (ir
->prev
< nk
|| (i
>= REF_BIAS
? ir
->prev
>= i
: ir
->prev
<= i
))
90 err
= "chain out of range";
91 else if (ir
->o
!= IR_NOP
&& IR(ir
->prev
)->o
!= ir
->o
)
92 err
= "chain to different op";
94 lj_assertJ(!err
, "bad IR %04d op %d(%04d,%04d): %s",
97 irm_op1(mode
) == IRMref
? op1
-REF_BIAS
: op1
,
98 irm_op2(mode
) == IRMref
? op2
-REF_BIAS
: op2
,
103 /* Compare stack slots and frames of the recorder and the VM. */
104 static void rec_check_slots(jit_State
*J
)
106 BCReg s
, nslots
= J
->baseslot
+ J
->maxslot
;
108 cTValue
*base
= J
->L
->base
- J
->baseslot
;
109 lj_assertJ(J
->baseslot
>= 1+LJ_FR2
, "bad baseslot");
110 lj_assertJ(J
->baseslot
== 1+LJ_FR2
|| (J
->slot
[J
->baseslot
-1] & TREF_FRAME
),
111 "baseslot does not point to frame");
112 lj_assertJ(nslots
<= LJ_MAX_JSLOTS
, "slot overflow");
113 for (s
= 0; s
< nslots
; s
++) {
114 TRef tr
= J
->slot
[s
];
116 cTValue
*tv
= &base
[s
];
117 IRRef ref
= tref_ref(tr
);
118 IRIns
*ir
= NULL
; /* Silence compiler. */
119 lj_assertJ(tv
< J
->L
->top
, "slot %d above top of Lua stack", s
);
120 if (!LJ_FR2
|| ref
|| !(tr
& (TREF_FRAME
| TREF_CONT
))) {
121 lj_assertJ(ref
>= J
->cur
.nk
&& ref
< J
->cur
.nins
,
122 "slot %d ref %04d out of range", s
, ref
- REF_BIAS
);
124 lj_assertJ(irt_t(ir
->t
) == tref_t(tr
), "slot %d IR type mismatch", s
);
127 lj_assertJ(tref_isfunc(tr
), "frame slot 0 is not a function");
130 lj_assertJ((tr
& ~TREF_FRAME
) == 0, "bad frame slot 1");
132 } else if ((tr
& TREF_FRAME
)) {
133 GCfunc
*fn
= gco2func(frame_gc(tv
));
134 BCReg delta
= (BCReg
)(tv
- frame_prev(tv
));
136 lj_assertJ(!ref
|| ir_knum(ir
)->u64
== tv
->u64
,
137 "frame slot %d PC mismatch", s
);
139 ir
= IR(tref_ref(tr
));
141 lj_assertJ(tref_isfunc(tr
),
142 "frame slot %d is not a function", s
-LJ_FR2
);
143 lj_assertJ(!tref_isk(tr
) || fn
== ir_kfunc(ir
),
144 "frame slot %d function mismatch", s
-LJ_FR2
);
145 lj_assertJ(s
> delta
+ LJ_FR2
? (J
->slot
[s
-delta
] & TREF_FRAME
)
146 : (s
== delta
+ LJ_FR2
),
147 "frame slot %d broken chain", s
-LJ_FR2
);
149 } else if ((tr
& TREF_CONT
)) {
151 lj_assertJ(!ref
|| ir_knum(ir
)->u64
== tv
->u64
,
152 "cont slot %d continuation mismatch", s
);
154 lj_assertJ(ir_kptr(ir
) == gcrefp(tv
->gcr
, void),
155 "cont slot %d continuation mismatch", s
);
157 lj_assertJ((J
->slot
[s
+1+LJ_FR2
] & TREF_FRAME
),
158 "cont slot %d not followed by frame", s
);
160 } else if ((tr
& TREF_KEYINDEX
)) {
161 lj_assertJ(tref_isint(tr
), "keyindex slot %d bad type %d",
164 /* Number repr. may differ, but other types must be the same. */
165 lj_assertJ(tvisnumber(tv
) ? tref_isnumber(tr
) :
166 itype2irt(tv
) == tref_type(tr
),
167 "slot %d type mismatch: stack type %d vs IR type %d",
168 s
, itypemap(tv
), tref_type(tr
));
169 if (tref_isk(tr
)) { /* Compare constants. */
171 lj_ir_kvalue(J
->L
, &tvk
, ir
);
172 lj_assertJ((tvisnum(&tvk
) && tvisnan(&tvk
)) ?
173 (tvisnum(tv
) && tvisnan(tv
)) :
174 lj_obj_equal(tv
, &tvk
),
175 "slot %d const mismatch: stack %016llx vs IR %016llx",
176 s
, tv
->u64
, tvk
.u64
);
181 lj_assertJ(J
->framedepth
== depth
,
182 "frame depth mismatch %d vs %d", J
->framedepth
, depth
);
186 /* -- Type handling and specialization ------------------------------------ */
188 /* Note: these functions return tagged references (TRef). */
190 /* Specialize a slot to a specific type. Note: slot can be negative! */
191 static TRef
sloadt(jit_State
*J
, int32_t slot
, IRType t
, int mode
)
193 /* Caller may set IRT_GUARD in t. */
194 TRef ref
= emitir_raw(IRT(IR_SLOAD
, t
), (int32_t)J
->baseslot
+slot
, mode
);
199 /* Specialize a slot to the runtime type. Note: slot can be negative! */
200 static TRef
sload(jit_State
*J
, int32_t slot
)
202 IRType t
= itype2irt(&J
->L
->base
[slot
]);
203 TRef ref
= emitir_raw(IRTG(IR_SLOAD
, t
), (int32_t)J
->baseslot
+slot
,
205 if (irtype_ispri(t
)) ref
= TREF_PRI(t
); /* Canonicalize primitive refs. */
210 /* Get TRef from slot. Load slot and specialize if not done already. */
211 #define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
213 /* Get TRef for current function. */
214 static TRef
getcurrf(jit_State
*J
)
216 if (J
->base
[-1-LJ_FR2
])
217 return J
->base
[-1-LJ_FR2
];
218 /* Non-base frame functions ought to be loaded already. */
219 lj_assertJ(J
->baseslot
== 1+LJ_FR2
, "bad baseslot");
220 return sloadt(J
, -1-LJ_FR2
, IRT_FUNC
, IRSLOAD_READONLY
);
223 /* Compare for raw object equality.
224 ** Returns 0 if the objects are the same.
225 ** Returns 1 if they are different, but the same type.
226 ** Returns 2 for two different types.
227 ** Comparisons between primitives always return 1 -- no caller cares about it.
229 int lj_record_objcmp(jit_State
*J
, TRef a
, TRef b
, cTValue
*av
, cTValue
*bv
)
231 int diff
= !lj_obj_equal(av
, bv
);
232 if (!tref_isk2(a
, b
)) { /* Shortcut, also handles primitives. */
233 IRType ta
= tref_isinteger(a
) ? IRT_INT
: tref_type(a
);
234 IRType tb
= tref_isinteger(b
) ? IRT_INT
: tref_type(b
);
236 /* Widen mixed number/int comparisons to number/number comparison. */
237 if (ta
== IRT_INT
&& tb
== IRT_NUM
) {
238 a
= emitir(IRTN(IR_CONV
), a
, IRCONV_NUM_INT
);
240 } else if (ta
== IRT_NUM
&& tb
== IRT_INT
) {
241 b
= emitir(IRTN(IR_CONV
), b
, IRCONV_NUM_INT
);
243 return 2; /* Two different types are never equal. */
246 emitir(IRTG(diff
? IR_NE
: IR_EQ
, ta
), a
, b
);
251 /* Constify a value. Returns 0 for non-representable object types. */
252 TRef
lj_record_constify(jit_State
*J
, cTValue
*o
)
255 return lj_ir_kgc(J
, gcV(o
), itype2irt(o
));
257 return lj_ir_kint(J
, intV(o
));
259 return lj_ir_knumint(J
, numV(o
));
260 else if (tvisbool(o
))
261 return TREF_PRI(itype2irt(o
));
263 return 0; /* Can't represent lightuserdata (pointless). */
266 /* Emit a VLOAD with the correct type. */
267 TRef
lj_record_vload(jit_State
*J
, TRef ref
, MSize idx
, IRType t
)
269 TRef tr
= emitir(IRTG(IR_VLOAD
, t
), ref
, idx
);
270 if (irtype_ispri(t
)) tr
= TREF_PRI(t
); /* Canonicalize primitives. */
274 /* -- Record loop ops ----------------------------------------------------- */
278 LOOPEV_LEAVE
, /* Loop is left or not entered. */
279 LOOPEV_ENTERLO
, /* Loop is entered with a low iteration count left. */
280 LOOPEV_ENTER
/* Loop is entered. */
283 /* Canonicalize slots: convert integers to numbers. */
284 static void canonicalize_slots(jit_State
*J
)
287 if (LJ_DUALNUM
) return;
288 for (s
= J
->baseslot
+J
->maxslot
-1; s
>= 1; s
--) {
289 TRef tr
= J
->slot
[s
];
290 if (tref_isinteger(tr
) && !(tr
& TREF_KEYINDEX
)) {
291 IRIns
*ir
= IR(tref_ref(tr
));
292 if (!(ir
->o
== IR_SLOAD
&& (ir
->op2
& (IRSLOAD_READONLY
))))
293 J
->slot
[s
] = emitir(IRTN(IR_CONV
), tr
, IRCONV_NUM_INT
);
298 /* Stop recording. */
299 void lj_record_stop(jit_State
*J
, TraceLink linktype
, TraceNo lnk
)
301 #ifdef LUAJIT_ENABLE_TABLE_BUMP
303 lj_trace_err(J
, LJ_TRERR_RETRY
);
306 J
->cur
.linktype
= (uint8_t)linktype
;
307 J
->cur
.link
= (uint16_t)lnk
;
308 /* Looping back at the same stack level? */
309 if (lnk
== J
->cur
.traceno
&& J
->framedepth
+ J
->retdepth
== 0) {
310 if ((J
->flags
& JIT_F_OPT_LOOP
)) /* Shall we try to create a loop? */
311 goto nocanon
; /* Do not canonicalize or we lose the narrowing. */
312 if (J
->cur
.root
) /* Otherwise ensure we always link to the root trace. */
313 J
->cur
.link
= J
->cur
.root
;
315 canonicalize_slots(J
);
317 /* Note: all loop ops must set J->pc to the following instruction! */
318 lj_snap_add(J
); /* Add loop snapshot. */
320 J
->mergesnap
= 1; /* In case recording continues. */
323 /* Search bytecode backwards for a int/num constant slot initializer. */
324 static TRef
find_kinit(jit_State
*J
, const BCIns
*endpc
, BCReg slot
, IRType t
)
326 /* This algorithm is rather simplistic and assumes quite a bit about
327 ** how the bytecode is generated. It works fine for FORI initializers,
328 ** but it won't necessarily work in other cases (e.g. iterator arguments).
329 ** It doesn't do anything fancy, either (like backpropagating MOVs).
331 const BCIns
*pc
, *startpc
= proto_bc(J
->pt
);
332 for (pc
= endpc
-1; pc
> startpc
; pc
--) {
334 BCOp op
= bc_op(ins
);
335 /* First try to find the last instruction that stores to this slot. */
336 if (bcmode_a(op
) == BCMbase
&& bc_a(ins
) <= slot
) {
337 return 0; /* Multiple results, e.g. from a CALL or KNIL. */
338 } else if (bcmode_a(op
) == BCMdst
&& bc_a(ins
) == slot
) {
339 if (op
== BC_KSHORT
|| op
== BC_KNUM
) { /* Found const. initializer. */
340 /* Now try to verify there's no forward jump across it. */
341 const BCIns
*kpc
= pc
;
342 for (; pc
> startpc
; pc
--)
343 if (bc_op(*pc
) == BC_JMP
) {
344 const BCIns
*target
= pc
+bc_j(*pc
)+1;
345 if (target
> kpc
&& target
<= endpc
)
346 return 0; /* Conditional assignment. */
348 if (op
== BC_KSHORT
) {
349 int32_t k
= (int32_t)(int16_t)bc_d(ins
);
350 return t
== IRT_INT
? lj_ir_kint(J
, k
) : lj_ir_knum(J
, (lua_Number
)k
);
352 cTValue
*tv
= proto_knumtv(J
->pt
, bc_d(ins
));
354 int32_t k
= numberVint(tv
);
355 if (tvisint(tv
) || numV(tv
) == (lua_Number
)k
) /* -0 is ok here. */
356 return lj_ir_kint(J
, k
);
357 return 0; /* Type mismatch. */
359 return lj_ir_knum(J
, numberVnum(tv
));
363 return 0; /* Non-constant initializer. */
366 return 0; /* No assignment to this slot found? */
369 /* Load and optionally convert a FORI argument from a slot. */
370 static TRef
fori_load(jit_State
*J
, BCReg slot
, IRType t
, int mode
)
372 int conv
= (tvisint(&J
->L
->base
[slot
]) != (t
==IRT_INT
)) ? IRSLOAD_CONVERT
: 0;
373 return sloadt(J
, (int32_t)slot
,
374 t
+ (((mode
& IRSLOAD_TYPECHECK
) ||
375 (conv
&& t
== IRT_INT
&& !(mode
>> 16))) ?
380 /* Peek before FORI to find a const initializer. Otherwise load from slot. */
381 static TRef
fori_arg(jit_State
*J
, const BCIns
*fori
, BCReg slot
,
384 TRef tr
= J
->base
[slot
];
386 tr
= find_kinit(J
, fori
, slot
, t
);
388 tr
= fori_load(J
, slot
, t
, mode
);
393 /* Return the direction of the FOR loop iterator.
394 ** It's important to exactly reproduce the semantics of the interpreter.
396 static int rec_for_direction(cTValue
*o
)
398 return (tvisint(o
) ? intV(o
) : (int32_t)o
->u32
.hi
) >= 0;
401 /* Simulate the runtime behavior of the FOR loop iterator. */
402 static LoopEvent
rec_for_iter(IROp
*op
, cTValue
*o
, int isforl
)
404 lua_Number stopv
= numberVnum(&o
[FORL_STOP
]);
405 lua_Number idxv
= numberVnum(&o
[FORL_IDX
]);
406 lua_Number stepv
= numberVnum(&o
[FORL_STEP
]);
409 if (rec_for_direction(&o
[FORL_STEP
])) {
412 return idxv
+ 2*stepv
> stopv
? LOOPEV_ENTERLO
: LOOPEV_ENTER
;
414 *op
= IR_GT
; return LOOPEV_LEAVE
;
418 return idxv
+ 2*stepv
< stopv
? LOOPEV_ENTERLO
: LOOPEV_ENTER
;
420 *op
= IR_LT
; return LOOPEV_LEAVE
;
424 /* Record checks for FOR loop overflow and step direction. */
425 static void rec_for_check(jit_State
*J
, IRType t
, int dir
,
426 TRef stop
, TRef step
, int init
)
428 if (!tref_isk(step
)) {
429 /* Non-constant step: need a guard for the direction. */
430 TRef zero
= (t
== IRT_INT
) ? lj_ir_kint(J
, 0) : lj_ir_knum_zero(J
);
431 emitir(IRTG(dir
? IR_GE
: IR_LT
, t
), step
, zero
);
432 /* Add hoistable overflow checks for a narrowed FORL index. */
433 if (init
&& t
== IRT_INT
) {
434 if (tref_isk(stop
)) {
435 /* Constant stop: optimize check away or to a range check for step. */
436 int32_t k
= IR(tref_ref(stop
))->i
;
439 emitir(IRTGI(IR_LE
), step
, lj_ir_kint(J
, (int32_t)0x7fffffff-k
));
442 emitir(IRTGI(IR_GE
), step
, lj_ir_kint(J
, (int32_t)0x80000000-k
));
445 /* Stop+step variable: need full overflow check. */
446 TRef tr
= emitir(IRTGI(IR_ADDOV
), step
, stop
);
447 emitir(IRTI(IR_USE
), tr
, 0); /* ADDOV is weak. Avoid dead result. */
450 } else if (init
&& t
== IRT_INT
&& !tref_isk(stop
)) {
451 /* Constant step: optimize overflow check to a range check for stop. */
452 int32_t k
= IR(tref_ref(step
))->i
;
453 k
= (int32_t)(dir
? 0x7fffffff : 0x80000000) - k
;
454 emitir(IRTGI(dir
? IR_LE
: IR_GE
), stop
, lj_ir_kint(J
, k
));
458 /* Record a FORL instruction. */
459 static void rec_for_loop(jit_State
*J
, const BCIns
*fori
, ScEvEntry
*scev
,
462 BCReg ra
= bc_a(*fori
);
463 cTValue
*tv
= &J
->L
->base
[ra
];
464 TRef idx
= J
->base
[ra
+FORL_IDX
];
465 IRType t
= idx
? tref_type(idx
) :
466 (init
|| LJ_DUALNUM
) ? lj_opt_narrow_forl(J
, tv
) : IRT_NUM
;
467 int mode
= IRSLOAD_INHERIT
+
468 ((!LJ_DUALNUM
|| tvisint(tv
) == (t
== IRT_INT
)) ? IRSLOAD_READONLY
: 0);
469 TRef stop
= fori_arg(J
, fori
, ra
+FORL_STOP
, t
, mode
);
470 TRef step
= fori_arg(J
, fori
, ra
+FORL_STEP
, t
, mode
);
471 int tc
, dir
= rec_for_direction(&tv
[FORL_STEP
]);
472 lj_assertJ(bc_op(*fori
) == BC_FORI
|| bc_op(*fori
) == BC_JFORI
,
473 "bad bytecode %d instead of FORI/JFORI", bc_op(*fori
));
476 scev
->stop
= tref_ref(stop
);
477 scev
->step
= tref_ref(step
);
478 rec_for_check(J
, t
, dir
, stop
, step
, init
);
479 scev
->start
= tref_ref(find_kinit(J
, fori
, ra
+FORL_IDX
, IRT_INT
));
481 !(scev
->start
&& irref_isk(scev
->stop
) && irref_isk(scev
->step
) &&
482 tvisint(&tv
[FORL_IDX
]) == (t
== IRT_INT
))) ?
483 IRSLOAD_TYPECHECK
: 0;
485 J
->base
[ra
+FORL_STOP
] = stop
;
486 J
->base
[ra
+FORL_STEP
] = step
;
489 idx
= fori_load(J
, ra
+FORL_IDX
, t
,
490 IRSLOAD_INHERIT
+ tc
+ (J
->scev
.start
<< 16));
492 J
->base
[ra
+FORL_IDX
] = idx
= emitir(IRT(IR_ADD
, t
), idx
, step
);
493 J
->base
[ra
+FORL_EXT
] = idx
;
494 scev
->idx
= tref_ref(idx
);
495 setmref(scev
->pc
, fori
);
496 J
->maxslot
= ra
+FORL_EXT
+1;
499 /* Record FORL/JFORL or FORI/JFORI. */
500 static LoopEvent
rec_for(jit_State
*J
, const BCIns
*fori
, int isforl
)
502 BCReg ra
= bc_a(*fori
);
503 TValue
*tv
= &J
->L
->base
[ra
];
504 TRef
*tr
= &J
->base
[ra
];
509 if (isforl
) { /* Handle FORL/JFORL opcodes. */
510 TRef idx
= tr
[FORL_IDX
];
511 if (mref(J
->scev
.pc
, const BCIns
) == fori
&& tref_ref(idx
) == J
->scev
.idx
) {
514 idx
= emitir(IRT(IR_ADD
, t
), idx
, J
->scev
.step
);
515 tr
[FORL_EXT
] = tr
[FORL_IDX
] = idx
;
518 rec_for_loop(J
, fori
, &scev
, 0);
522 } else { /* Handle FORI/JFORI opcodes. */
524 lj_meta_for(J
->L
, tv
);
525 t
= (LJ_DUALNUM
|| tref_isint(tr
[FORL_IDX
])) ? lj_opt_narrow_forl(J
, tv
) :
527 for (i
= FORL_IDX
; i
<= FORL_STEP
; i
++) {
528 if (!tr
[i
]) sload(J
, ra
+i
);
529 lj_assertJ(tref_isnumber_str(tr
[i
]), "bad FORI argument type");
530 if (tref_isstr(tr
[i
]))
531 tr
[i
] = emitir(IRTG(IR_STRTO
, IRT_NUM
), tr
[i
], 0);
533 if (!tref_isinteger(tr
[i
]))
534 tr
[i
] = emitir(IRTGI(IR_CONV
), tr
[i
], IRCONV_INT_NUM
|IRCONV_CHECK
);
536 if (!tref_isnum(tr
[i
]))
537 tr
[i
] = emitir(IRTN(IR_CONV
), tr
[i
], IRCONV_NUM_INT
);
540 tr
[FORL_EXT
] = tr
[FORL_IDX
];
541 stop
= tr
[FORL_STOP
];
542 rec_for_check(J
, t
, rec_for_direction(&tv
[FORL_STEP
]),
543 stop
, tr
[FORL_STEP
], 1);
546 ev
= rec_for_iter(&op
, tv
, isforl
);
547 if (ev
== LOOPEV_LEAVE
) {
548 J
->maxslot
= ra
+FORL_EXT
+1;
552 J
->pc
= fori
+bc_j(*fori
)+1;
556 emitir(IRTG(op
, t
), tr
[FORL_IDX
], stop
);
558 if (ev
== LOOPEV_LEAVE
) {
560 J
->pc
= fori
+bc_j(*fori
)+1;
562 J
->maxslot
= ra
+FORL_EXT
+1;
569 /* Record ITERL/JITERL. */
570 static LoopEvent
rec_iterl(jit_State
*J
, const BCIns iterins
)
572 BCReg ra
= bc_a(iterins
);
573 if (!tref_isnil(getslot(J
, ra
))) { /* Looping back? */
574 J
->base
[ra
-1] = J
->base
[ra
]; /* Copy result of ITERC to control var. */
575 J
->maxslot
= ra
-1+bc_b(J
->pc
[-1]);
576 J
->pc
+= bc_j(iterins
)+1;
585 /* Record LOOP/JLOOP. Now, that was easy. */
586 static LoopEvent
rec_loop(jit_State
*J
, BCReg ra
, int skip
)
588 if (ra
< J
->maxslot
) J
->maxslot
= ra
;
593 /* Check if a loop repeatedly failed to trace because it didn't loop back. */
594 static int innerloopleft(jit_State
*J
, const BCIns
*pc
)
597 for (i
= 0; i
< PENALTY_SLOTS
; i
++)
598 if (mref(J
->penalty
[i
].pc
, const BCIns
) == pc
) {
599 if ((J
->penalty
[i
].reason
== LJ_TRERR_LLEAVE
||
600 J
->penalty
[i
].reason
== LJ_TRERR_LINNER
) &&
601 J
->penalty
[i
].val
>= 2*PENALTY_MIN
)
608 /* Handle the case when an interpreted loop op is hit. */
609 static void rec_loop_interp(jit_State
*J
, const BCIns
*pc
, LoopEvent ev
)
611 if (J
->parent
== 0 && J
->exitno
== 0) {
612 if (pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0) {
613 if (bc_op(J
->cur
.startins
) == BC_ITERN
) return; /* See rec_itern(). */
615 if (ev
== LOOPEV_LEAVE
) /* Must loop back to form a root trace. */
616 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
617 lj_record_stop(J
, LJ_TRLINK_LOOP
, J
->cur
.traceno
); /* Looping trace. */
618 } else if (ev
!= LOOPEV_LEAVE
) { /* Entering inner loop? */
619 /* It's usually better to abort here and wait until the inner loop
620 ** is traced. But if the inner loop repeatedly didn't loop back,
621 ** this indicates a low trip count. In this case try unrolling
622 ** an inner loop even in a root trace. But it's better to be a bit
623 ** more conservative here and only do it for very short loops.
625 if (bc_j(*pc
) != -1 && !innerloopleft(J
, pc
))
626 lj_trace_err(J
, LJ_TRERR_LINNER
); /* Root trace hit an inner loop. */
627 if ((ev
!= LOOPEV_ENTERLO
&&
628 J
->loopref
&& J
->cur
.nins
- J
->loopref
> 24) || --J
->loopunroll
< 0)
629 lj_trace_err(J
, LJ_TRERR_LUNROLL
); /* Limit loop unrolling. */
630 J
->loopref
= J
->cur
.nins
;
632 } else if (ev
!= LOOPEV_LEAVE
) { /* Side trace enters an inner loop. */
633 J
->loopref
= J
->cur
.nins
;
634 if (--J
->loopunroll
< 0)
635 lj_trace_err(J
, LJ_TRERR_LUNROLL
); /* Limit loop unrolling. */
636 } /* Side trace continues across a loop that's left or not entered. */
639 /* Handle the case when an already compiled loop op is hit. */
640 static void rec_loop_jit(jit_State
*J
, TraceNo lnk
, LoopEvent ev
)
642 if (J
->parent
== 0 && J
->exitno
== 0) { /* Root trace hit an inner loop. */
643 /* Better let the inner loop spawn a side trace back here. */
644 lj_trace_err(J
, LJ_TRERR_LINNER
);
645 } else if (ev
!= LOOPEV_LEAVE
) { /* Side trace enters a compiled loop. */
646 J
->instunroll
= 0; /* Cannot continue across a compiled loop op. */
647 if (J
->pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0)
648 lj_record_stop(J
, LJ_TRLINK_LOOP
, J
->cur
.traceno
); /* Form extra loop. */
650 lj_record_stop(J
, LJ_TRLINK_ROOT
, lnk
); /* Link to the loop. */
651 } /* Side trace continues across a loop that's left or not entered. */
655 static LoopEvent
rec_itern(jit_State
*J
, BCReg ra
, BCReg rb
)
658 /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
659 ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
661 UNUSED(ra
); UNUSED(rb
);
662 setintV(&J
->errinfo
, (int32_t)BC_ITERN
);
663 lj_trace_err_info(J
, LJ_TRERR_NYIBC
);
666 /* Since ITERN is recorded at the start, we need our own loop detection. */
667 if (J
->pc
== J
->startpc
&&
668 J
->framedepth
+ J
->retdepth
== 0 && J
->parent
== 0 && J
->exitno
== 0) {
669 IRRef ref
= REF_FIRST
+ LJ_HASPROFILE
;
670 #ifdef LUAJIT_ENABLE_CHECKHOOK
673 if (J
->cur
.nins
> ref
||
674 (LJ_HASPROFILE
&& J
->cur
.nins
== ref
&& J
->cur
.ir
[ref
-1].o
!= IR_PROF
)) {
675 J
->instunroll
= 0; /* Cannot continue unrolling across an ITERN. */
676 lj_record_stop(J
, LJ_TRLINK_LOOP
, J
->cur
.traceno
); /* Looping trace. */
681 lj_snap_add(J
); /* Required to make JLOOP the first ins in a side-trace. */
682 ix
.tab
= getslot(J
, ra
-2);
683 ix
.key
= J
->base
[ra
-1] ? J
->base
[ra
-1] :
684 sloadt(J
, (int32_t)(ra
-1), IRT_GUARD
|IRT_INT
,
685 IRSLOAD_TYPECHECK
|IRSLOAD_KEYINDEX
);
686 copyTV(J
->L
, &ix
.tabv
, &J
->L
->base
[ra
-2]);
687 copyTV(J
->L
, &ix
.keyv
, &J
->L
->base
[ra
-1]);
688 ix
.idxchain
= (rb
< 3); /* Omit value type check, if unused. */
689 ix
.mobj
= 1; /* We need the next index, too. */
690 J
->maxslot
= ra
+ lj_record_next(J
, &ix
);
692 if (!tref_isnil(ix
.key
)) { /* Looping back? */
693 J
->base
[ra
-1] = ix
.mobj
| TREF_KEYINDEX
; /* Control var has next index. */
694 J
->base
[ra
] = ix
.key
;
695 J
->base
[ra
+1] = ix
.val
;
696 J
->pc
+= bc_j(J
->pc
[1])+2;
707 static void rec_isnext(jit_State
*J
, BCReg ra
)
709 cTValue
*b
= &J
->L
->base
[ra
-3];
710 if (tvisfunc(b
) && funcV(b
)->c
.ffid
== FF_next
&&
711 tvistab(b
+1) && tvisnil(b
+2)) {
712 /* These checks are folded away for a compiled pairs(). */
713 TRef func
= getslot(J
, ra
-3);
714 TRef trid
= emitir(IRT(IR_FLOAD
, IRT_U8
), func
, IRFL_FUNC_FFID
);
715 emitir(IRTGI(IR_EQ
), trid
, lj_ir_kint(J
, FF_next
));
716 (void)getslot(J
, ra
-2); /* Type check for table. */
717 (void)getslot(J
, ra
-1); /* Type check for nil key. */
718 J
->base
[ra
-1] = lj_ir_kint(J
, 0) | TREF_KEYINDEX
;
720 } else { /* Abort trace. Interpreter will despecialize bytecode. */
721 lj_trace_err(J
, LJ_TRERR_RECERR
);
725 /* -- Record profiler hook checks ----------------------------------------- */
729 /* Need to insert profiler hook check? */
730 static int rec_profile_need(jit_State
*J
, GCproto
*pt
, const BCIns
*pc
)
733 lj_assertJ(J
->prof_mode
== 'f' || J
->prof_mode
== 'l',
734 "bad profiler mode %c", J
->prof_mode
);
739 if (pt
!= ppt
&& ppt
) {
743 if (J
->prof_mode
== 'l') {
744 BCLine line
= lj_debug_line(pt
, proto_bcpos(pt
, pc
));
745 BCLine pline
= J
->prev_line
;
753 static void rec_profile_ins(jit_State
*J
, const BCIns
*pc
)
755 if (J
->prof_mode
&& rec_profile_need(J
, J
->pt
, pc
)) {
756 emitir(IRTG(IR_PROF
, IRT_NIL
), 0, 0);
761 static void rec_profile_ret(jit_State
*J
)
763 if (J
->prof_mode
== 'f') {
764 emitir(IRTG(IR_PROF
, IRT_NIL
), 0, 0);
772 /* -- Record calls and returns -------------------------------------------- */
774 /* Specialize to the runtime value of the called function or its prototype. */
775 static TRef
rec_call_specialize(jit_State
*J
, GCfunc
*fn
, TRef tr
)
779 GCproto
*pt
= funcproto(fn
);
780 /* Too many closures created? Probably not a monomorphic function. */
781 if (pt
->flags
>= PROTO_CLC_POLY
) { /* Specialize to prototype instead. */
782 TRef trpt
= emitir(IRT(IR_FLOAD
, IRT_PGC
), tr
, IRFL_FUNC_PC
);
783 emitir(IRTG(IR_EQ
, IRT_PGC
), trpt
, lj_ir_kptr(J
, proto_bc(pt
)));
784 (void)lj_ir_kgc(J
, obj2gco(pt
), IRT_PROTO
); /* Prevent GC of proto. */
788 /* Don't specialize to non-monomorphic builtins. */
789 switch (fn
->c
.ffid
) {
790 case FF_coroutine_wrap_aux
:
791 case FF_string_gmatch_aux
:
792 /* NYI: io_file_iter doesn't have an ffid, yet. */
793 { /* Specialize to the ffid. */
794 TRef trid
= emitir(IRT(IR_FLOAD
, IRT_U8
), tr
, IRFL_FUNC_FFID
);
795 emitir(IRTGI(IR_EQ
), trid
, lj_ir_kint(J
, fn
->c
.ffid
));
799 /* NYI: don't specialize to non-monomorphic C functions. */
803 /* Otherwise specialize to the function (closure) value itself. */
804 kfunc
= lj_ir_kfunc(J
, fn
);
805 emitir(IRTG(IR_EQ
, IRT_FUNC
), tr
, kfunc
);
809 /* Record call setup. */
810 static void rec_call_setup(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
813 TValue
*functv
= &J
->L
->base
[func
];
814 TRef kfunc
, *fbase
= &J
->base
[func
];
816 (void)getslot(J
, func
); /* Ensure func has a reference. */
817 for (i
= 1; i
<= nargs
; i
++)
818 (void)getslot(J
, func
+LJ_FR2
+i
); /* Ensure all args have a reference. */
819 if (!tref_isfunc(fbase
[0])) { /* Resolve __call metamethod. */
821 copyTV(J
->L
, &ix
.tabv
, functv
);
822 if (!lj_record_mm_lookup(J
, &ix
, MM_call
) || !tref_isfunc(ix
.mobj
))
823 lj_trace_err(J
, LJ_TRERR_NOMM
);
824 for (i
= ++nargs
; i
> LJ_FR2
; i
--) /* Shift arguments up. */
825 fbase
[i
+LJ_FR2
] = fbase
[i
+LJ_FR2
-1];
829 fbase
[0] = ix
.mobj
; /* Replace function. */
832 kfunc
= rec_call_specialize(J
, funcV(functv
), fbase
[0]);
835 fbase
[1] = TREF_FRAME
;
837 fbase
[0] = kfunc
| TREF_FRAME
;
839 J
->maxslot
= (BCReg
)nargs
;
843 void lj_record_call(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
845 rec_call_setup(J
, func
, nargs
);
848 J
->base
+= func
+1+LJ_FR2
;
849 J
->baseslot
+= func
+1+LJ_FR2
;
850 if (J
->baseslot
+ J
->maxslot
>= LJ_MAX_JSLOTS
)
851 lj_trace_err(J
, LJ_TRERR_STACKOV
);
854 /* Record tail call. */
855 void lj_record_tailcall(jit_State
*J
, BCReg func
, ptrdiff_t nargs
)
857 rec_call_setup(J
, func
, nargs
);
858 if (frame_isvarg(J
->L
->base
- 1)) {
859 BCReg cbase
= (BCReg
)frame_delta(J
->L
->base
- 1);
860 if (--J
->framedepth
< 0)
861 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
862 J
->baseslot
-= (BCReg
)cbase
;
866 /* Move func + args down. */
867 if (LJ_FR2
&& J
->baseslot
== 2)
868 J
->base
[func
+1] = TREF_FRAME
;
869 memmove(&J
->base
[-1-LJ_FR2
], &J
->base
[func
], sizeof(TRef
)*(J
->maxslot
+1+LJ_FR2
));
870 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
871 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
872 if (++J
->tailcalled
> J
->loopunroll
)
873 lj_trace_err(J
, LJ_TRERR_LUNROLL
);
876 /* Check unroll limits for down-recursion. */
877 static int check_downrec_unroll(jit_State
*J
, GCproto
*pt
)
880 for (ptref
= J
->chain
[IR_KGC
]; ptref
; ptref
= IR(ptref
)->prev
)
881 if (ir_kgc(IR(ptref
)) == obj2gco(pt
)) {
884 for (ref
= J
->chain
[IR_RETF
]; ref
; ref
= IR(ref
)->prev
)
885 if (IR(ref
)->op1
== ptref
)
888 if (J
->pc
== J
->startpc
) {
889 if (count
+ J
->tailcalled
> J
->param
[JIT_P_recunroll
])
892 lj_trace_err(J
, LJ_TRERR_DOWNREC
);
899 static TRef
rec_cat(jit_State
*J
, BCReg baseslot
, BCReg topslot
);
902 void lj_record_ret(jit_State
*J
, BCReg rbase
, ptrdiff_t gotresults
)
904 TValue
*frame
= J
->L
->base
- 1;
906 for (i
= 0; i
< gotresults
; i
++)
907 (void)getslot(J
, rbase
+i
); /* Ensure all results have a reference. */
908 while (frame_ispcall(frame
)) { /* Immediately resolve pcall() returns. */
909 BCReg cbase
= (BCReg
)frame_delta(frame
);
910 if (--J
->framedepth
<= 0)
911 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
912 lj_assertJ(J
->baseslot
> 1+LJ_FR2
, "bad baseslot for return");
915 J
->baseslot
-= (BCReg
)cbase
;
917 J
->base
[--rbase
] = TREF_TRUE
; /* Prepend true to results. */
918 frame
= frame_prevd(frame
);
919 J
->needsnap
= 1; /* Stop catching on-trace errors. */
921 /* Return to lower frame via interpreter for unhandled cases. */
922 if (J
->framedepth
== 0 && J
->pt
&& bc_isret(bc_op(*J
->pc
)) &&
923 (!frame_islua(frame
) ||
924 (J
->parent
== 0 && J
->exitno
== 0 &&
925 !bc_isret(bc_op(J
->cur
.startins
))))) {
926 /* NYI: specialize to frame type and return directly, not via RET*. */
927 for (i
= 0; i
< (ptrdiff_t)rbase
; i
++)
928 J
->base
[i
] = 0; /* Purge dead slots. */
929 J
->maxslot
= rbase
+ (BCReg
)gotresults
;
930 lj_record_stop(J
, LJ_TRLINK_RETURN
, 0); /* Return to interpreter. */
933 if (frame_isvarg(frame
)) {
934 BCReg cbase
= (BCReg
)frame_delta(frame
);
935 if (--J
->framedepth
< 0) /* NYI: return of vararg func to lower frame. */
936 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
937 lj_assertJ(J
->baseslot
> 1+LJ_FR2
, "bad baseslot for return");
939 J
->baseslot
-= (BCReg
)cbase
;
941 frame
= frame_prevd(frame
);
943 if (frame_islua(frame
)) { /* Return to Lua frame. */
944 BCIns callins
= *(frame_pc(frame
)-1);
945 ptrdiff_t nresults
= bc_b(callins
) ? (ptrdiff_t)bc_b(callins
)-1 :gotresults
;
946 BCReg cbase
= bc_a(callins
);
947 GCproto
*pt
= funcproto(frame_func(frame
- (cbase
+1+LJ_FR2
)));
948 if ((pt
->flags
& PROTO_NOJIT
))
949 lj_trace_err(J
, LJ_TRERR_CJITOFF
);
950 if (J
->framedepth
== 0 && J
->pt
&& frame
== J
->L
->base
- 1) {
951 if (check_downrec_unroll(J
, pt
)) {
952 J
->maxslot
= (BCReg
)(rbase
+ gotresults
);
954 lj_record_stop(J
, LJ_TRLINK_DOWNREC
, J
->cur
.traceno
); /* Down-rec. */
959 for (i
= 0; i
< nresults
; i
++) /* Adjust results. */
960 J
->base
[i
-1-LJ_FR2
] = i
< gotresults
? J
->base
[rbase
+i
] : TREF_NIL
;
961 J
->maxslot
= cbase
+(BCReg
)nresults
;
962 if (J
->framedepth
> 0) { /* Return to a frame that is part of the trace. */
964 lj_assertJ(J
->baseslot
> cbase
+1+LJ_FR2
, "bad baseslot for return");
965 J
->baseslot
-= cbase
+1+LJ_FR2
;
966 J
->base
-= cbase
+1+LJ_FR2
;
967 } else if (J
->parent
== 0 && J
->exitno
== 0 &&
968 !bc_isret(bc_op(J
->cur
.startins
))) {
969 /* Return to lower frame would leave the loop in a root trace. */
970 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
971 } else if (J
->needsnap
) { /* Tailcalled to ff with side-effects. */
972 lj_trace_err(J
, LJ_TRERR_NYIRETL
); /* No way to insert snapshot here. */
973 } else { /* Return to lower frame. Guard for the target we return to. */
974 TRef trpt
= lj_ir_kgc(J
, obj2gco(pt
), IRT_PROTO
);
975 TRef trpc
= lj_ir_kptr(J
, (void *)frame_pc(frame
));
976 emitir(IRTG(IR_RETF
, IRT_PGC
), trpt
, trpc
);
979 J
->scev
.idx
= REF_NIL
;
980 lj_assertJ(J
->baseslot
== 1+LJ_FR2
, "bad baseslot for return");
981 /* Shift result slots up and clear the slots of the new frame below. */
982 memmove(J
->base
+ cbase
, J
->base
-1-LJ_FR2
, sizeof(TRef
)*nresults
);
983 memset(J
->base
-1-LJ_FR2
, 0, sizeof(TRef
)*(cbase
+1+LJ_FR2
));
985 } else if (frame_iscont(frame
)) { /* Return to continuation frame. */
986 ASMFunction cont
= frame_contf(frame
);
987 BCReg cbase
= (BCReg
)frame_delta(frame
);
988 if ((J
->framedepth
-= 2) < 0)
989 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
990 J
->baseslot
-= (BCReg
)cbase
;
992 J
->maxslot
= cbase
-(2<<LJ_FR2
);
993 if (cont
== lj_cont_ra
) {
994 /* Copy result to destination slot. */
995 BCReg dst
= bc_a(*(frame_contpc(frame
)-1));
996 J
->base
[dst
] = gotresults
? J
->base
[cbase
+rbase
] : TREF_NIL
;
997 if (dst
>= J
->maxslot
) {
1000 } else if (cont
== lj_cont_nop
) {
1001 /* Nothing to do here. */
1002 } else if (cont
== lj_cont_cat
) {
1003 BCReg bslot
= bc_b(*(frame_contpc(frame
)-1));
1004 TRef tr
= gotresults
? J
->base
[cbase
+rbase
] : TREF_NIL
;
1005 if (bslot
!= J
->maxslot
) { /* Concatenate the remainder. */
1006 TValue
*b
= J
->L
->base
, save
; /* Simulate lower frame and result. */
1007 /* Can't handle MM_concat + CALLT + fast func side-effects. */
1008 if (J
->postproc
!= LJ_POST_NONE
)
1009 lj_trace_err(J
, LJ_TRERR_NYIRETL
);
1010 J
->base
[J
->maxslot
] = tr
;
1011 copyTV(J
->L
, &save
, b
-(2<<LJ_FR2
));
1013 copyTV(J
->L
, b
-(2<<LJ_FR2
), b
+rbase
);
1015 setnilV(b
-(2<<LJ_FR2
));
1016 J
->L
->base
= b
- cbase
;
1017 tr
= rec_cat(J
, bslot
, cbase
-(2<<LJ_FR2
));
1018 b
= J
->L
->base
+ cbase
; /* Undo. */
1020 copyTV(J
->L
, b
-(2<<LJ_FR2
), &save
);
1022 if (tr
) { /* Store final result. */
1023 BCReg dst
= bc_a(*(frame_contpc(frame
)-1));
1025 if (dst
>= J
->maxslot
) {
1028 } /* Otherwise continue with another __concat call. */
1030 /* Result type already specialized. */
1031 lj_assertJ(cont
== lj_cont_condf
|| cont
== lj_cont_condt
,
1032 "bad continuation type");
1035 lj_trace_err(J
, LJ_TRERR_NYIRETL
); /* NYI: handle return to C frame. */
1037 lj_assertJ(J
->baseslot
>= 1+LJ_FR2
, "bad baseslot for return");
1040 /* -- Metamethod handling ------------------------------------------------- */
1042 /* Prepare to record call to metamethod. */
1043 static BCReg
rec_mm_prep(jit_State
*J
, ASMFunction cont
)
1045 BCReg s
, top
= cont
== lj_cont_cat
? J
->maxslot
: curr_proto(J
->L
)->framesize
;
1047 J
->base
[top
] = lj_ir_k64(J
, IR_KNUM
, u64ptr(contptr(cont
)));
1048 J
->base
[top
+1] = TREF_CONT
;
1050 J
->base
[top
] = lj_ir_kptr(J
, contptr(cont
)) | TREF_CONT
;
1053 for (s
= J
->maxslot
; s
< top
; s
++)
1054 J
->base
[s
] = 0; /* Clear frame gap to avoid resurrecting previous refs. */
1055 return top
+1+LJ_FR2
;
1058 /* Record metamethod lookup. */
1059 int lj_record_mm_lookup(jit_State
*J
, RecordIndex
*ix
, MMS mm
)
1063 if (tref_istab(ix
->tab
)) {
1064 mt
= tabref(tabV(&ix
->tabv
)->metatable
);
1065 mix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_TAB_META
);
1066 } else if (tref_isudata(ix
->tab
)) {
1067 int udtype
= udataV(&ix
->tabv
)->udtype
;
1068 mt
= tabref(udataV(&ix
->tabv
)->metatable
);
1069 /* The metatables of special userdata objects are treated as immutable. */
1070 if (udtype
!= UDTYPE_USERDATA
) {
1072 if (LJ_HASFFI
&& udtype
== UDTYPE_FFI_CLIB
) {
1073 /* Specialize to the C library namespace object. */
1074 emitir(IRTG(IR_EQ
, IRT_PGC
), ix
->tab
, lj_ir_kptr(J
, udataV(&ix
->tabv
)));
1076 /* Specialize to the type of userdata. */
1077 TRef tr
= emitir(IRT(IR_FLOAD
, IRT_U8
), ix
->tab
, IRFL_UDATA_UDTYPE
);
1078 emitir(IRTGI(IR_EQ
), tr
, lj_ir_kint(J
, udtype
));
1081 mo
= lj_tab_getstr(mt
, mmname_str(J2G(J
), mm
));
1082 if (!mo
|| tvisnil(mo
))
1083 return 0; /* No metamethod. */
1084 /* Treat metamethod or index table as immutable, too. */
1085 if (!(tvisfunc(mo
) || tvistab(mo
)))
1086 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1087 copyTV(J
->L
, &ix
->mobjv
, mo
);
1088 ix
->mobj
= lj_ir_kgc(J
, gcV(mo
), tvisfunc(mo
) ? IRT_FUNC
: IRT_TAB
);
1090 ix
->mt
= TREF_NIL
; /* Dummy value for comparison semantics. */
1091 return 1; /* Got metamethod or index table. */
1093 mix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_UDATA_META
);
1095 /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
1096 mt
= tabref(basemt_obj(J2G(J
), &ix
->tabv
));
1099 return 0; /* No metamethod. */
1101 /* The cdata metatable is treated as immutable. */
1102 if (LJ_HASFFI
&& tref_iscdata(ix
->tab
)) goto immutable_mt
;
1103 ix
->mt
= mix
.tab
= lj_ir_ggfload(J
, IRT_TAB
,
1104 GG_OFS(g
.gcroot
[GCROOT_BASEMT
+itypemap(&ix
->tabv
)]));
1107 ix
->mt
= mt
? mix
.tab
: TREF_NIL
;
1108 emitir(IRTG(mt
? IR_NE
: IR_EQ
, IRT_TAB
), mix
.tab
, lj_ir_knull(J
, IRT_TAB
));
1111 GCstr
*mmstr
= mmname_str(J2G(J
), mm
);
1112 cTValue
*mo
= lj_tab_getstr(mt
, mmstr
);
1113 if (mo
&& !tvisnil(mo
))
1114 copyTV(J
->L
, &ix
->mobjv
, mo
);
1116 settabV(J
->L
, &mix
.tabv
, mt
);
1117 setstrV(J
->L
, &mix
.keyv
, mmstr
);
1118 mix
.key
= lj_ir_kstr(J
, mmstr
);
1121 ix
->mobj
= lj_record_idx(J
, &mix
);
1122 return !tref_isnil(ix
->mobj
); /* 1 if metamethod found, 0 if not. */
1124 return 0; /* No metamethod. */
1127 /* Record call to arithmetic metamethod. */
1128 static TRef
rec_mm_arith(jit_State
*J
, RecordIndex
*ix
, MMS mm
)
1130 /* Set up metamethod call first to save ix->tab and ix->tabv. */
1131 BCReg func
= rec_mm_prep(J
, mm
== MM_concat
? lj_cont_cat
: lj_cont_ra
);
1132 TRef
*base
= J
->base
+ func
;
1133 TValue
*basev
= J
->L
->base
+ func
;
1134 base
[1+LJ_FR2
] = ix
->tab
; base
[2+LJ_FR2
] = ix
->key
;
1135 copyTV(J
->L
, basev
+1+LJ_FR2
, &ix
->tabv
);
1136 copyTV(J
->L
, basev
+2+LJ_FR2
, &ix
->keyv
);
1137 if (!lj_record_mm_lookup(J
, ix
, mm
)) { /* Lookup mm on 1st operand. */
1140 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
1141 if (lj_record_mm_lookup(J
, ix
, mm
)) /* Lookup mm on 2nd operand. */
1144 lj_trace_err(J
, LJ_TRERR_NOMM
);
1151 copyTV(J
->L
, basev
+0, &ix
->mobjv
);
1152 lj_record_call(J
, func
, 2);
1153 return 0; /* No result yet. */
1156 /* Record call to __len metamethod. */
1157 static TRef
rec_mm_len(jit_State
*J
, TRef tr
, TValue
*tv
)
1161 copyTV(J
->L
, &ix
.tabv
, tv
);
1162 if (lj_record_mm_lookup(J
, &ix
, MM_len
)) {
1163 BCReg func
= rec_mm_prep(J
, lj_cont_ra
);
1164 TRef
*base
= J
->base
+ func
;
1165 TValue
*basev
= J
->L
->base
+ func
;
1166 base
[0] = ix
.mobj
; copyTV(J
->L
, basev
+0, &ix
.mobjv
);
1169 base
[1] = tr
; copyTV(J
->L
, basev
+1, tv
);
1171 base
[2] = tr
; copyTV(J
->L
, basev
+2, tv
);
1173 base
[2] = TREF_NIL
; setnilV(basev
+2);
1175 lj_record_call(J
, func
, 2);
1177 if (LJ_52
&& tref_istab(tr
))
1178 return emitir(IRTI(IR_ALEN
), tr
, TREF_NIL
);
1179 lj_trace_err(J
, LJ_TRERR_NOMM
);
1181 return 0; /* No result yet. */
1184 /* Call a comparison metamethod. */
1185 static void rec_mm_callcomp(jit_State
*J
, RecordIndex
*ix
, int op
)
1187 BCReg func
= rec_mm_prep(J
, (op
&1) ? lj_cont_condf
: lj_cont_condt
);
1188 TRef
*base
= J
->base
+ func
+ LJ_FR2
;
1189 TValue
*tv
= J
->L
->base
+ func
+ LJ_FR2
;
1190 base
[-LJ_FR2
] = ix
->mobj
; base
[1] = ix
->val
; base
[2] = ix
->key
;
1191 copyTV(J
->L
, tv
-LJ_FR2
, &ix
->mobjv
);
1192 copyTV(J
->L
, tv
+1, &ix
->valv
);
1193 copyTV(J
->L
, tv
+2, &ix
->keyv
);
1194 lj_record_call(J
, func
, 2);
1197 /* Record call to equality comparison metamethod (for tab and udata only). */
1198 static void rec_mm_equal(jit_State
*J
, RecordIndex
*ix
, int op
)
1201 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
1202 if (lj_record_mm_lookup(J
, ix
, MM_eq
)) { /* Lookup mm on 1st operand. */
1204 TRef mo1
= ix
->mobj
;
1206 copyTV(J
->L
, &mo1v
, &ix
->mobjv
);
1207 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1209 if (tvistab(bv
) && tabref(tabV(bv
)->metatable
) == ix
->mtv
) {
1210 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_TAB_META
);
1211 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
1212 } else if (tvisudata(bv
) && tabref(udataV(bv
)->metatable
) == ix
->mtv
) {
1213 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_UDATA_META
);
1214 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
1215 } else { /* Lookup metamethod on 2nd operand and compare both. */
1217 copyTV(J
->L
, &ix
->tabv
, bv
);
1218 if (!lj_record_mm_lookup(J
, ix
, MM_eq
) ||
1219 lj_record_objcmp(J
, mo1
, ix
->mobj
, &mo1v
, &ix
->mobjv
))
1222 rec_mm_callcomp(J
, ix
, op
);
1226 /* Record call to ordered comparison metamethods (for arbitrary objects). */
1227 static void rec_mm_comp(jit_State
*J
, RecordIndex
*ix
, int op
)
1230 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
1232 MMS mm
= (op
& 2) ? MM_le
: MM_lt
; /* Try __le + __lt or only __lt. */
1234 if (!lj_record_mm_lookup(J
, ix
, mm
)) { /* Lookup mm on 1st operand. */
1236 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
1237 if (!lj_record_mm_lookup(J
, ix
, mm
)) /* Lookup mm on 2nd operand. */
1240 rec_mm_callcomp(J
, ix
, op
);
1243 if (lj_record_mm_lookup(J
, ix
, mm
)) { /* Lookup mm on 1st operand. */
1245 TRef mo1
= ix
->mobj
;
1247 copyTV(J
->L
, &mo1v
, &ix
->mobjv
);
1248 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1250 if (tvistab(bv
) && tabref(tabV(bv
)->metatable
) == ix
->mtv
) {
1251 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_TAB_META
);
1252 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
1253 } else if (tvisudata(bv
) && tabref(udataV(bv
)->metatable
) == ix
->mtv
) {
1254 TRef mt2
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->key
, IRFL_UDATA_META
);
1255 emitir(IRTG(IR_EQ
, IRT_TAB
), mt2
, ix
->mt
);
1256 } else { /* Lookup metamethod on 2nd operand and compare both. */
1258 copyTV(J
->L
, &ix
->tabv
, bv
);
1259 if (!lj_record_mm_lookup(J
, ix
, mm
) ||
1260 lj_record_objcmp(J
, mo1
, ix
->mobj
, &mo1v
, &ix
->mobjv
))
1263 rec_mm_callcomp(J
, ix
, op
);
1268 /* Lookup failed. Retry with __lt and swapped operands. */
1269 if (!(op
& 2)) break; /* Already at __lt. Interpreter will throw. */
1270 ix
->tab
= ix
->key
; ix
->key
= ix
->val
; ix
->val
= ix
->tab
;
1271 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
1272 copyTV(J
->L
, &ix
->keyv
, &ix
->valv
);
1273 copyTV(J
->L
, &ix
->valv
, &ix
->tabv
);
1279 /* Setup call to cdata comparison metamethod. */
1280 static void rec_mm_comp_cdata(jit_State
*J
, RecordIndex
*ix
, int op
, MMS mm
)
1283 if (tref_iscdata(ix
->val
)) {
1285 copyTV(J
->L
, &ix
->tabv
, &ix
->valv
);
1287 lj_assertJ(tref_iscdata(ix
->key
), "cdata expected");
1289 copyTV(J
->L
, &ix
->tabv
, &ix
->keyv
);
1291 lj_record_mm_lookup(J
, ix
, mm
);
1292 rec_mm_callcomp(J
, ix
, op
);
1296 /* -- Indexed access ------------------------------------------------------ */
1298 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1299 /* Bump table allocations in bytecode when they grow during recording. */
1300 static void rec_idx_bump(jit_State
*J
, RecordIndex
*ix
)
1302 RBCHashEntry
*rbc
= &J
->rbchash
[(ix
->tab
& (RBCHASH_SLOTS
-1))];
1303 if (tref_ref(ix
->tab
) == rbc
->ref
) {
1304 const BCIns
*pc
= mref(rbc
->pc
, const BCIns
);
1305 GCtab
*tb
= tabV(&ix
->tabv
);
1308 if (!tvisnil(&ix
->keyv
))
1309 (void)lj_tab_set(J
->L
, tb
, &ix
->keyv
); /* Grow table right now. */
1310 nhbits
= tb
->hmask
> 0 ? lj_fls(tb
->hmask
)+1 : 0;
1311 ir
= IR(tref_ref(ix
->tab
));
1312 if (ir
->o
== IR_TNEW
) {
1313 uint32_t ah
= bc_d(*pc
);
1314 uint32_t asize
= ah
& 0x7ff, hbits
= ah
>> 11;
1315 if (nhbits
> hbits
) hbits
= nhbits
;
1316 if (tb
->asize
> asize
) {
1317 asize
= tb
->asize
<= 0x7ff ? tb
->asize
: 0x7ff;
1319 if ((asize
| (hbits
<<11)) != ah
) { /* Has the size changed? */
1320 /* Patch bytecode, but continue recording (for more patching). */
1321 setbc_d(pc
, (asize
| (hbits
<<11)));
1322 /* Patching TNEW operands is only safe if the trace is aborted. */
1323 ir
->op1
= asize
; ir
->op2
= hbits
;
1324 J
->retryrec
= 1; /* Abort the trace at the end of recording. */
1326 } else if (ir
->o
== IR_TDUP
) {
1327 GCtab
*tpl
= gco2tab(proto_kgc(&gcref(rbc
->pt
)->pt
, ~(ptrdiff_t)bc_d(*pc
)));
1328 /* Grow template table, but preserve keys with nil values. */
1329 if ((tb
->asize
> tpl
->asize
&& (1u << nhbits
)-1 == tpl
->hmask
) ||
1330 (tb
->asize
== tpl
->asize
&& (1u << nhbits
)-1 > tpl
->hmask
)) {
1331 Node
*node
= noderef(tpl
->node
);
1332 uint32_t i
, hmask
= tpl
->hmask
, asize
;
1334 for (i
= 0; i
<= hmask
; i
++) {
1335 if (!tvisnil(&node
[i
].key
) && tvisnil(&node
[i
].val
))
1336 settabV(J
->L
, &node
[i
].val
, tpl
);
1338 if (!tvisnil(&ix
->keyv
) && tref_isk(ix
->key
)) {
1339 TValue
*o
= lj_tab_set(J
->L
, tpl
, &ix
->keyv
);
1340 if (tvisnil(o
)) settabV(J
->L
, o
, tpl
);
1342 lj_tab_resize(J
->L
, tpl
, tb
->asize
, nhbits
);
1343 node
= noderef(tpl
->node
);
1345 for (i
= 0; i
<= hmask
; i
++) {
1346 /* This is safe, since template tables only hold immutable values. */
1347 if (tvistab(&node
[i
].val
))
1348 setnilV(&node
[i
].val
);
1350 /* The shape of the table may have changed. Clean up array part, too. */
1352 array
= tvref(tpl
->array
);
1353 for (i
= 0; i
< asize
; i
++) {
1354 if (tvistab(&array
[i
]))
1357 J
->retryrec
= 1; /* Abort the trace at the end of recording. */
1364 /* Record bounds-check. */
1365 static void rec_idx_abc(jit_State
*J
, TRef asizeref
, TRef ikey
, uint32_t asize
)
1367 /* Try to emit invariant bounds checks. */
1368 if ((J
->flags
& (JIT_F_OPT_LOOP
|JIT_F_OPT_ABC
)) ==
1369 (JIT_F_OPT_LOOP
|JIT_F_OPT_ABC
)) {
1370 IRRef ref
= tref_ref(ikey
);
1371 IRIns
*ir
= IR(ref
);
1374 /* Handle constant offsets. */
1375 if (ir
->o
== IR_ADD
&& irref_isk(ir
->op2
)) {
1377 ofs
= IR(ofsref
)->i
;
1381 /* Got scalar evolution analysis results for this reference? */
1382 if (ref
== J
->scev
.idx
) {
1384 lj_assertJ(irt_isint(J
->scev
.t
) && ir
->o
== IR_SLOAD
,
1385 "only int SCEV supported");
1386 stop
= numberVint(&(J
->L
->base
- J
->baseslot
)[ir
->op1
+ FORL_STOP
]);
1387 /* Runtime value for stop of loop is within bounds? */
1388 if ((uint64_t)stop
+ ofs
< (uint64_t)asize
) {
1389 /* Emit invariant bounds check for stop. */
1390 emitir(IRTG(IR_ABC
, IRT_P32
), asizeref
, ofs
== 0 ? J
->scev
.stop
:
1391 emitir(IRTI(IR_ADD
), J
->scev
.stop
, ofsref
));
1392 /* Emit invariant bounds check for start, if not const or negative. */
1393 if (!(J
->scev
.dir
&& J
->scev
.start
&&
1394 (int64_t)IR(J
->scev
.start
)->i
+ ofs
>= 0))
1395 emitir(IRTG(IR_ABC
, IRT_P32
), asizeref
, ikey
);
1400 emitir(IRTGI(IR_ABC
), asizeref
, ikey
); /* Emit regular bounds check. */
1403 /* Record indexed key lookup. */
1404 static TRef
rec_idx_key(jit_State
*J
, RecordIndex
*ix
, IRRef
*rbref
,
1408 GCtab
*t
= tabV(&ix
->tabv
);
1409 ix
->oldv
= lj_tab_get(J
->L
, t
, &ix
->keyv
); /* Lookup previous value. */
1413 /* Integer keys are looked up in the array part first. */
1415 if (tref_isnumber(key
)) {
1416 int32_t k
= numberVint(&ix
->keyv
);
1417 if (!tvisint(&ix
->keyv
) && numV(&ix
->keyv
) != (lua_Number
)k
)
1419 if ((MSize
)k
< LJ_MAX_ASIZE
) { /* Potential array key? */
1420 TRef ikey
= lj_opt_narrow_index(J
, key
);
1421 TRef asizeref
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_ASIZE
);
1422 if ((MSize
)k
< t
->asize
) { /* Currently an array key? */
1424 rec_idx_abc(J
, asizeref
, ikey
, t
->asize
);
1425 arrayref
= emitir(IRT(IR_FLOAD
, IRT_PGC
), ix
->tab
, IRFL_TAB_ARRAY
);
1426 return emitir(IRT(IR_AREF
, IRT_PGC
), arrayref
, ikey
);
1427 } else { /* Currently not in array (may be an array extension)? */
1428 emitir(IRTGI(IR_ULE
), asizeref
, ikey
); /* Inv. bounds check. */
1429 if (k
== 0 && tref_isk(key
))
1430 key
= lj_ir_knum_zero(J
); /* Canonicalize 0 or +-0.0 to +0.0. */
1431 /* And continue with the hash lookup. */
1433 } else if (!tref_isk(key
)) {
1434 /* We can rule out const numbers which failed the integerness test
1435 ** above. But all other numbers are potential array keys.
1437 if (t
->asize
== 0) { /* True sparse tables have an empty array part. */
1438 /* Guard that the array part stays empty. */
1439 TRef tmp
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_ASIZE
);
1440 emitir(IRTGI(IR_EQ
), tmp
, lj_ir_kint(J
, 0));
1442 lj_trace_err(J
, LJ_TRERR_NYITMIX
);
1447 /* Otherwise the key is located in the hash part. */
1448 if (t
->hmask
== 0) { /* Shortcut for empty hash part. */
1449 /* Guard that the hash part stays empty. */
1450 TRef tmp
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_HMASK
);
1451 emitir(IRTGI(IR_EQ
), tmp
, lj_ir_kint(J
, 0));
1452 return lj_ir_kkptr(J
, niltvg(J2G(J
)));
1454 if (tref_isinteger(key
)) /* Hash keys are based on numbers, not ints. */
1455 key
= emitir(IRTN(IR_CONV
), key
, IRCONV_NUM_INT
);
1456 if (tref_isk(key
)) {
1457 /* Optimize lookup of constant hash keys. */
1458 GCSize hslot
= (GCSize
)((char *)ix
->oldv
-(char *)&noderef(t
->node
)[0].val
);
1459 if (hslot
<= t
->hmask
*(GCSize
)sizeof(Node
) &&
1460 hslot
<= 65535*(GCSize
)sizeof(Node
)) {
1461 TRef node
, kslot
, hm
;
1462 *rbref
= J
->cur
.nins
; /* Mark possible rollback point. */
1463 *rbguard
= J
->guardemit
;
1464 hm
= emitir(IRTI(IR_FLOAD
), ix
->tab
, IRFL_TAB_HMASK
);
1465 emitir(IRTGI(IR_EQ
), hm
, lj_ir_kint(J
, (int32_t)t
->hmask
));
1466 node
= emitir(IRT(IR_FLOAD
, IRT_PGC
), ix
->tab
, IRFL_TAB_NODE
);
1467 kslot
= lj_ir_kslot(J
, key
, (IRRef
)(hslot
/ sizeof(Node
)));
1468 return emitir(IRTG(IR_HREFK
, IRT_PGC
), node
, kslot
);
1471 /* Fall back to a regular hash lookup. */
1472 return emitir(IRT(IR_HREF
, IRT_PGC
), ix
->tab
, key
);
1475 /* Determine whether a key is NOT one of the fast metamethod names. */
1476 static int nommstr(jit_State
*J
, TRef key
)
1478 if (tref_isstr(key
)) {
1479 if (tref_isk(key
)) {
1480 GCstr
*str
= ir_kstr(IR(tref_ref(key
)));
1482 for (mm
= 0; mm
<= MM_FAST
; mm
++)
1483 if (mmname_str(J2G(J
), mm
) == str
)
1484 return 0; /* MUST be one the fast metamethod names. */
1486 return 0; /* Variable string key MAY be a metamethod name. */
1489 return 1; /* CANNOT be a metamethod name. */
1492 /* Record indexed load/store. */
1493 TRef
lj_record_idx(jit_State
*J
, RecordIndex
*ix
)
1496 IROp xrefop
, loadop
;
1501 while (!tref_istab(ix
->tab
)) { /* Handle non-table lookup. */
1502 /* Never call raw lj_record_idx() on non-table. */
1503 lj_assertJ(ix
->idxchain
!= 0, "bad usage");
1504 if (!lj_record_mm_lookup(J
, ix
, ix
->val
? MM_newindex
: MM_index
))
1505 lj_trace_err(J
, LJ_TRERR_NOMM
);
1507 if (tref_isfunc(ix
->mobj
)) { /* Handle metamethod call. */
1508 BCReg func
= rec_mm_prep(J
, ix
->val
? lj_cont_nop
: lj_cont_ra
);
1509 TRef
*base
= J
->base
+ func
+ LJ_FR2
;
1510 TValue
*tv
= J
->L
->base
+ func
+ LJ_FR2
;
1511 base
[-LJ_FR2
] = ix
->mobj
; base
[1] = ix
->tab
; base
[2] = ix
->key
;
1512 setfuncV(J
->L
, tv
-LJ_FR2
, funcV(&ix
->mobjv
));
1513 copyTV(J
->L
, tv
+1, &ix
->tabv
);
1514 copyTV(J
->L
, tv
+2, &ix
->keyv
);
1517 copyTV(J
->L
, tv
+3, &ix
->valv
);
1518 lj_record_call(J
, func
, 3); /* mobj(tab, key, val) */
1521 lj_record_call(J
, func
, 2); /* res = mobj(tab, key) */
1522 return 0; /* No result yet. */
1526 /* The index table of buffer objects is treated as immutable. */
1527 if (ix
->mt
== TREF_NIL
&& !ix
->val
&&
1528 tref_isudata(ix
->tab
) && udataV(&ix
->tabv
)->udtype
== UDTYPE_BUFFER
&&
1529 tref_istab(ix
->mobj
) && tref_isstr(ix
->key
) && tref_isk(ix
->key
)) {
1530 cTValue
*val
= lj_tab_getstr(tabV(&ix
->mobjv
), strV(&ix
->keyv
));
1531 TRef tr
= lj_record_constify(J
, val
);
1532 if (tr
) return tr
; /* Specialize to the value, i.e. a method. */
1535 /* Otherwise retry lookup with metaobject. */
1537 copyTV(J
->L
, &ix
->tabv
, &ix
->mobjv
);
1538 if (--ix
->idxchain
== 0)
1539 lj_trace_err(J
, LJ_TRERR_IDXLOOP
);
1542 /* First catch nil and NaN keys for tables. */
1543 if (tvisnil(&ix
->keyv
) || (tvisnum(&ix
->keyv
) && tvisnan(&ix
->keyv
))) {
1544 if (ix
->val
) /* Better fail early. */
1545 lj_trace_err(J
, LJ_TRERR_STORENN
);
1546 if (tref_isk(ix
->key
)) {
1547 if (ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_index
))
1553 /* Record the key lookup. */
1554 xref
= rec_idx_key(J
, ix
, &rbref
, &rbguard
);
1555 xrefop
= IR(tref_ref(xref
))->o
;
1556 loadop
= xrefop
== IR_AREF
? IR_ALOAD
: IR_HLOAD
;
1557 /* The lj_meta_tset() inconsistency is gone, but better play safe. */
1558 oldv
= xrefop
== IR_KKPTR
? (cTValue
*)ir_kptr(IR(tref_ref(xref
))) : ix
->oldv
;
1560 if (ix
->val
== 0) { /* Indexed load */
1561 IRType t
= itype2irt(oldv
);
1563 if (oldv
== niltvg(J2G(J
))) {
1564 emitir(IRTG(IR_EQ
, IRT_PGC
), xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1567 res
= emitir(IRTG(loadop
, t
), xref
, 0);
1569 if (tref_ref(res
) < rbref
) { /* HREFK + load forwarded? */
1570 lj_ir_rollback(J
, rbref
); /* Rollback to eliminate hmask guard. */
1571 J
->guardemit
= rbguard
;
1573 if (t
== IRT_NIL
&& ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_index
))
1575 if (irtype_ispri(t
)) res
= TREF_PRI(t
); /* Canonicalize primitives. */
1577 } else { /* Indexed store. */
1578 GCtab
*mt
= tabref(tabV(&ix
->tabv
)->metatable
);
1579 int keybarrier
= tref_isgcv(ix
->key
) && !tref_isnil(ix
->val
);
1580 if (tref_ref(xref
) < rbref
) { /* HREFK forwarded? */
1581 lj_ir_rollback(J
, rbref
); /* Rollback to eliminate hmask guard. */
1582 J
->guardemit
= rbguard
;
1584 if (tvisnil(oldv
)) { /* Previous value was nil? */
1585 /* Need to duplicate the hasmm check for the early guards. */
1587 if (ix
->idxchain
&& mt
) {
1588 cTValue
*mo
= lj_tab_getstr(mt
, mmname_str(J2G(J
), MM_newindex
));
1589 hasmm
= mo
&& !tvisnil(mo
);
1592 emitir(IRTG(loadop
, IRT_NIL
), xref
, 0); /* Guard for nil value. */
1593 else if (xrefop
== IR_HREF
)
1594 emitir(IRTG(oldv
== niltvg(J2G(J
)) ? IR_EQ
: IR_NE
, IRT_PGC
),
1595 xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1596 if (ix
->idxchain
&& lj_record_mm_lookup(J
, ix
, MM_newindex
)) {
1597 lj_assertJ(hasmm
, "inconsistent metamethod handling");
1600 lj_assertJ(!hasmm
, "inconsistent metamethod handling");
1601 if (oldv
== niltvg(J2G(J
))) { /* Need to insert a new key. */
1603 if (tref_isinteger(key
)) { /* NEWREF needs a TValue as a key. */
1604 key
= emitir(IRTN(IR_CONV
), key
, IRCONV_NUM_INT
);
1605 } else if (tref_isnum(key
)) {
1606 if (tref_isk(key
)) {
1607 if (tvismzero(&ix
->keyv
))
1608 key
= lj_ir_knum_zero(J
); /* Canonicalize -0.0 to +0.0. */
1610 emitir(IRTG(IR_EQ
, IRT_NUM
), key
, key
); /* Check for !NaN. */
1613 xref
= emitir(IRT(IR_NEWREF
, IRT_PGC
), ix
->tab
, key
);
1614 keybarrier
= 0; /* NEWREF already takes care of the key barrier. */
1615 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1616 if ((J
->flags
& JIT_F_OPT_SINK
)) /* Avoid a separate flag. */
1617 rec_idx_bump(J
, ix
);
1620 } else if (!lj_opt_fwd_wasnonnil(J
, loadop
, tref_ref(xref
))) {
1621 /* Cannot derive that the previous value was non-nil, must do checks. */
1622 if (xrefop
== IR_HREF
) /* Guard against store to niltv. */
1623 emitir(IRTG(IR_NE
, IRT_PGC
), xref
, lj_ir_kkptr(J
, niltvg(J2G(J
))));
1624 if (ix
->idxchain
) { /* Metamethod lookup required? */
1625 /* A check for NULL metatable is cheaper (hoistable) than a load. */
1627 TRef mtref
= emitir(IRT(IR_FLOAD
, IRT_TAB
), ix
->tab
, IRFL_TAB_META
);
1628 emitir(IRTG(IR_EQ
, IRT_TAB
), mtref
, lj_ir_knull(J
, IRT_TAB
));
1630 IRType t
= itype2irt(oldv
);
1631 emitir(IRTG(loadop
, t
), xref
, 0); /* Guard for non-nil value. */
1635 keybarrier
= 0; /* Previous non-nil value kept the key alive. */
1637 /* Convert int to number before storing. */
1638 if (!LJ_DUALNUM
&& tref_isinteger(ix
->val
))
1639 ix
->val
= emitir(IRTN(IR_CONV
), ix
->val
, IRCONV_NUM_INT
);
1640 emitir(IRT(loadop
+IRDELTA_L2S
, tref_type(ix
->val
)), xref
, ix
->val
);
1641 if (keybarrier
|| tref_isgcv(ix
->val
))
1642 emitir(IRT(IR_TBAR
, IRT_NIL
), ix
->tab
, 0);
1643 /* Invalidate neg. metamethod cache for stores with certain string keys. */
1644 if (!nommstr(J
, ix
->key
)) {
1645 TRef fref
= emitir(IRT(IR_FREF
, IRT_PGC
), ix
->tab
, IRFL_TAB_NOMM
);
1646 emitir(IRT(IR_FSTORE
, IRT_U8
), fref
, lj_ir_kint(J
, 0));
1653 /* Determine result type of table traversal. */
1654 static IRType
rec_next_types(GCtab
*t
, uint32_t idx
)
1656 for (; idx
< t
->asize
; idx
++) {
1657 cTValue
*a
= arrayslot(t
, idx
);
1658 if (LJ_LIKELY(!tvisnil(a
)))
1659 return (LJ_DUALNUM
? IRT_INT
: IRT_NUM
) + (itype2irt(a
) << 8);
1662 for (; idx
<= t
->hmask
; idx
++) {
1663 Node
*n
= &noderef(t
->node
)[idx
];
1664 if (!tvisnil(&n
->val
))
1665 return itype2irt(&n
->key
) + (itype2irt(&n
->val
) << 8);
1667 return IRT_NIL
+ (IRT_NIL
<< 8);
1670 /* Record a table traversal step aka next(). */
1671 int lj_record_next(jit_State
*J
, RecordIndex
*ix
)
1673 IRType t
, tkey
, tval
;
1675 t
= rec_next_types(tabV(&ix
->tabv
), ix
->keyv
.u32
.lo
);
1676 tkey
= (t
& 0xff); tval
= (t
>> 8);
1677 trvk
= lj_ir_call(J
, IRCALL_lj_vm_next
, ix
->tab
, ix
->key
);
1678 if (ix
->mobj
|| tkey
== IRT_NIL
) {
1679 TRef idx
= emitir(IRTI(IR_HIOP
), trvk
, trvk
);
1680 /* Always check for invalid key from next() for nil result. */
1681 if (!ix
->mobj
) emitir(IRTGI(IR_NE
), idx
, lj_ir_kint(J
, -1));
1684 ix
->key
= lj_record_vload(J
, trvk
, 1, tkey
);
1685 if (tkey
== IRT_NIL
|| ix
->idxchain
) { /* Omit value type check. */
1688 } else { /* Need value. */
1689 ix
->val
= lj_record_vload(J
, trvk
, 0, tval
);
1694 static void rec_tsetm(jit_State
*J
, BCReg ra
, BCReg rn
, int32_t i
)
1697 cTValue
*basev
= J
->L
->base
;
1698 GCtab
*t
= tabV(&basev
[ra
-1]);
1699 settabV(J
->L
, &ix
.tabv
, t
);
1700 ix
.tab
= getslot(J
, ra
-1);
1702 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1703 if ((J
->flags
& JIT_F_OPT_SINK
)) {
1704 if (t
->asize
< i
+rn
-ra
)
1705 lj_tab_reasize(J
->L
, t
, i
+rn
-ra
);
1707 rec_idx_bump(J
, &ix
);
1710 for (; ra
< rn
; i
++, ra
++) {
1711 setintV(&ix
.keyv
, i
);
1712 ix
.key
= lj_ir_kint(J
, i
);
1713 copyTV(J
->L
, &ix
.valv
, &basev
[ra
]);
1714 ix
.val
= getslot(J
, ra
);
1715 lj_record_idx(J
, &ix
);
1719 /* -- Upvalue access ------------------------------------------------------ */
1721 /* Check whether upvalue is immutable and ok to constify. */
1722 static int rec_upvalue_constify(jit_State
*J
, GCupval
*uvp
)
1724 if (uvp
->immutable
) {
1725 cTValue
*o
= uvval(uvp
);
1726 /* Don't constify objects that may retain large amounts of memory. */
1729 GCcdata
*cd
= cdataV(o
);
1730 if (!cdataisv(cd
) && !(cd
->marked
& LJ_GC_CDATA_FIN
)) {
1731 CType
*ct
= ctype_raw(ctype_ctsG(J2G(J
)), cd
->ctypeid
);
1732 if (!ctype_hassize(ct
->info
) || ct
->size
<= 16)
1740 if (!(tvistab(o
) || tvisudata(o
) || tvisthread(o
)))
1746 /* Record upvalue load/store. */
1747 static TRef
rec_upvalue(jit_State
*J
, uint32_t uv
, TRef val
)
1749 GCupval
*uvp
= &gcref(J
->fn
->l
.uvptr
[uv
])->uv
;
1750 TRef fn
= getcurrf(J
);
1752 int needbarrier
= 0;
1753 if (rec_upvalue_constify(J
, uvp
)) { /* Try to constify immutable upvalue. */
1755 lj_assertJ(val
== 0, "bad usage");
1756 if (!tref_isk(fn
)) { /* Late specialization of current function. */
1757 if (J
->pt
->flags
>= PROTO_CLC_POLY
)
1759 kfunc
= lj_ir_kfunc(J
, J
->fn
);
1760 emitir(IRTG(IR_EQ
, IRT_FUNC
), fn
, kfunc
);
1762 J
->base
[-2] = kfunc
;
1764 J
->base
[-1] = kfunc
| TREF_FRAME
;
1768 tr
= lj_record_constify(J
, uvval(uvp
));
1773 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1774 uv
= (uv
<< 8) | (hashrot(uvp
->dhash
, uvp
->dhash
+ HASH_BIAS
) & 0xff);
1776 /* In current stack? */
1777 if (uvval(uvp
) >= tvref(J
->L
->stack
) &&
1778 uvval(uvp
) < tvref(J
->L
->maxstack
)) {
1779 int32_t slot
= (int32_t)(uvval(uvp
) - (J
->L
->base
- J
->baseslot
));
1780 if (slot
>= 0) { /* Aliases an SSA slot? */
1781 uref
= tref_ref(emitir(IRT(IR_UREFO
, IRT_PGC
), fn
, uv
));
1782 emitir(IRTG(IR_EQ
, IRT_PGC
),
1784 emitir(IRT(IR_ADD
, IRT_PGC
), uref
,
1785 lj_ir_kintpgc(J
, (slot
- 1 - LJ_FR2
) * -8)));
1786 slot
-= (int32_t)J
->baseslot
; /* Note: slot number may be negative! */
1788 return getslot(J
, slot
);
1790 J
->base
[slot
] = val
;
1791 if (slot
>= (int32_t)J
->maxslot
) J
->maxslot
= (BCReg
)(slot
+1);
1796 /* IR_UREFO+IRT_IGC is not checked for open-ness at runtime.
1797 ** Always marked as a guard, since it might get promoted to IRT_PGC later.
1799 uref
= emitir(IRTG(IR_UREFO
, tref_isgcv(val
) ? IRT_PGC
: IRT_IGC
), fn
, uv
);
1800 uref
= tref_ref(uref
);
1801 emitir(IRTG(IR_UGT
, IRT_PGC
),
1802 emitir(IRT(IR_SUB
, IRT_PGC
), uref
, REF_BASE
),
1803 lj_ir_kintpgc(J
, (J
->baseslot
+ J
->maxslot
) * 8));
1805 /* If fn is constant, then so is the GCupval*, and the upvalue cannot
1806 ** transition back to open, so no guard is required in this case.
1808 IRType t
= (tref_isk(fn
) ? 0 : IRT_GUARD
) | IRT_PGC
;
1809 uref
= tref_ref(emitir(IRT(IR_UREFC
, t
), fn
, uv
));
1812 if (val
== 0) { /* Upvalue load */
1813 IRType t
= itype2irt(uvval(uvp
));
1814 TRef res
= emitir(IRTG(IR_ULOAD
, t
), uref
, 0);
1815 if (irtype_ispri(t
)) res
= TREF_PRI(t
); /* Canonicalize primitive refs. */
1817 } else { /* Upvalue store. */
1818 /* Convert int to number before storing. */
1819 if (!LJ_DUALNUM
&& tref_isinteger(val
))
1820 val
= emitir(IRTN(IR_CONV
), val
, IRCONV_NUM_INT
);
1821 emitir(IRT(IR_USTORE
, tref_type(val
)), uref
, val
);
1822 if (needbarrier
&& tref_isgcv(val
))
1823 emitir(IRT(IR_OBAR
, IRT_NIL
), uref
, val
);
1829 /* -- Record calls to Lua functions --------------------------------------- */
1831 /* Check unroll limits for calls. */
1832 static void check_call_unroll(jit_State
*J
, TraceNo lnk
)
1834 cTValue
*frame
= J
->L
->base
- 1;
1835 void *pc
= mref(frame_func(frame
)->l
.pc
, void);
1836 int32_t depth
= J
->framedepth
;
1838 if ((J
->pt
->flags
& PROTO_VARARG
)) depth
--; /* Vararg frame still missing. */
1839 for (; depth
> 0; depth
--) { /* Count frames with same prototype. */
1840 if (frame_iscont(frame
)) depth
--;
1841 frame
= frame_prev(frame
);
1842 if (mref(frame_func(frame
)->l
.pc
, void) == pc
)
1845 if (J
->pc
== J
->startpc
) {
1846 if (count
+ J
->tailcalled
> J
->param
[JIT_P_recunroll
]) {
1848 if (J
->framedepth
+ J
->retdepth
== 0)
1849 lj_record_stop(J
, LJ_TRLINK_TAILREC
, J
->cur
.traceno
); /* Tail-rec. */
1851 lj_record_stop(J
, LJ_TRLINK_UPREC
, J
->cur
.traceno
); /* Up-recursion. */
1854 if (count
> J
->param
[JIT_P_callunroll
]) {
1855 if (lnk
) { /* Possible tail- or up-recursion. */
1856 lj_trace_flush(J
, lnk
); /* Flush trace that only returns. */
1857 /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1858 hotcount_set(J2GG(J
), J
->pc
+1, lj_prng_u64(&J2G(J
)->prng
) & 15u);
1860 lj_trace_err(J
, LJ_TRERR_CUNROLL
);
1865 /* Record Lua function setup. */
1866 static void rec_func_setup(jit_State
*J
)
1868 GCproto
*pt
= J
->pt
;
1869 BCReg s
, numparams
= pt
->numparams
;
1870 if ((pt
->flags
& PROTO_NOJIT
))
1871 lj_trace_err(J
, LJ_TRERR_CJITOFF
);
1872 if (J
->baseslot
+ pt
->framesize
>= LJ_MAX_JSLOTS
)
1873 lj_trace_err(J
, LJ_TRERR_STACKOV
);
1874 /* Fill up missing parameters with nil. */
1875 for (s
= J
->maxslot
; s
< numparams
; s
++)
1876 J
->base
[s
] = TREF_NIL
;
1877 /* The remaining slots should never be read before they are written. */
1878 J
->maxslot
= numparams
;
1881 /* Record Lua vararg function setup. */
1882 static void rec_func_vararg(jit_State
*J
)
1884 GCproto
*pt
= J
->pt
;
1885 BCReg s
, fixargs
, vframe
= J
->maxslot
+1+LJ_FR2
;
1886 lj_assertJ((pt
->flags
& PROTO_VARARG
), "FUNCV in non-vararg function");
1887 if (J
->baseslot
+ vframe
+ pt
->framesize
>= LJ_MAX_JSLOTS
)
1888 lj_trace_err(J
, LJ_TRERR_STACKOV
);
1889 J
->base
[vframe
-1-LJ_FR2
] = J
->base
[-1-LJ_FR2
]; /* Copy function up. */
1891 J
->base
[vframe
-1] = TREF_FRAME
;
1893 /* Copy fixarg slots up and set their original slots to nil. */
1894 fixargs
= pt
->numparams
< J
->maxslot
? pt
->numparams
: J
->maxslot
;
1895 for (s
= 0; s
< fixargs
; s
++) {
1896 J
->base
[vframe
+s
] = J
->base
[s
];
1897 J
->base
[s
] = TREF_NIL
;
1899 J
->maxslot
= fixargs
;
1902 J
->baseslot
+= vframe
;
1905 /* Record entry to a Lua function. */
1906 static void rec_func_lua(jit_State
*J
)
1909 check_call_unroll(J
, 0);
1912 /* Record entry to an already compiled function. */
1913 static void rec_func_jit(jit_State
*J
, TraceNo lnk
)
1917 T
= traceref(J
, lnk
);
1918 if (T
->linktype
== LJ_TRLINK_RETURN
) { /* Trace returns to interpreter? */
1919 check_call_unroll(J
, lnk
);
1920 /* Temporarily unpatch JFUNC* to continue recording across function. */
1921 J
->patchins
= *J
->pc
;
1922 J
->patchpc
= (BCIns
*)J
->pc
;
1923 *J
->patchpc
= T
->startins
;
1926 J
->instunroll
= 0; /* Cannot continue across a compiled function. */
1927 if (J
->pc
== J
->startpc
&& J
->framedepth
+ J
->retdepth
== 0)
1928 lj_record_stop(J
, LJ_TRLINK_TAILREC
, J
->cur
.traceno
); /* Extra tail-rec. */
1930 lj_record_stop(J
, LJ_TRLINK_ROOT
, lnk
); /* Link to the function. */
1933 /* -- Vararg handling ----------------------------------------------------- */
1935 /* Detect y = select(x, ...) idiom. */
1936 static int select_detect(jit_State
*J
)
1938 BCIns ins
= J
->pc
[1];
1939 if (bc_op(ins
) == BC_CALLM
&& bc_b(ins
) == 2 && bc_c(ins
) == 1) {
1940 cTValue
*func
= &J
->L
->base
[bc_a(ins
)];
1941 if (tvisfunc(func
) && funcV(func
)->c
.ffid
== FF_select
) {
1942 TRef kfunc
= lj_ir_kfunc(J
, funcV(func
));
1943 emitir(IRTG(IR_EQ
, IRT_FUNC
), getslot(J
, bc_a(ins
)), kfunc
);
1950 /* Record vararg instruction. */
1951 static void rec_varg(jit_State
*J
, BCReg dst
, ptrdiff_t nresults
)
1953 int32_t numparams
= J
->pt
->numparams
;
1954 ptrdiff_t nvararg
= frame_delta(J
->L
->base
-1) - numparams
- 1 - LJ_FR2
;
1955 lj_assertJ(frame_isvarg(J
->L
->base
-1), "VARG in non-vararg frame");
1956 if (LJ_FR2
&& dst
> J
->maxslot
)
1957 J
->base
[dst
-1] = 0; /* Prevent resurrection of unrelated slot. */
1958 if (J
->framedepth
> 0) { /* Simple case: varargs defined on-trace. */
1960 if (nvararg
< 0) nvararg
= 0;
1961 if (nresults
!= 1) {
1962 if (nresults
== -1) nresults
= nvararg
;
1963 J
->maxslot
= dst
+ (BCReg
)nresults
;
1964 } else if (dst
>= J
->maxslot
) {
1965 J
->maxslot
= dst
+ 1;
1967 if (J
->baseslot
+ J
->maxslot
>= LJ_MAX_JSLOTS
)
1968 lj_trace_err(J
, LJ_TRERR_STACKOV
);
1969 for (i
= 0; i
< nresults
; i
++)
1970 J
->base
[dst
+i
] = i
< nvararg
? getslot(J
, i
- nvararg
- 1 - LJ_FR2
) : TREF_NIL
;
1971 } else { /* Unknown number of varargs passed to trace. */
1972 TRef fr
= emitir(IRTI(IR_SLOAD
), LJ_FR2
, IRSLOAD_READONLY
|IRSLOAD_FRAME
);
1973 int32_t frofs
= 8*(1+LJ_FR2
+numparams
)+FRAME_VARG
;
1974 if (nresults
>= 0) { /* Known fixed number of results. */
1977 ptrdiff_t nload
= nvararg
>= nresults
? nresults
: nvararg
;
1979 if (nvararg
>= nresults
)
1980 emitir(IRTGI(IR_GE
), fr
, lj_ir_kint(J
, frofs
+8*(int32_t)nresults
));
1982 emitir(IRTGI(IR_EQ
), fr
,
1983 lj_ir_kint(J
, (int32_t)frame_ftsz(J
->L
->base
-1)));
1984 vbase
= emitir(IRT(IR_SUB
, IRT_IGC
), REF_BASE
, fr
);
1985 vbase
= emitir(IRT(IR_ADD
, IRT_PGC
), vbase
,
1986 lj_ir_kintpgc(J
, frofs
-8*(1+LJ_FR2
)));
1987 for (i
= 0; i
< nload
; i
++) {
1988 IRType t
= itype2irt(&J
->L
->base
[i
-1-LJ_FR2
-nvararg
]);
1989 J
->base
[dst
+i
] = lj_record_vload(J
, vbase
, (MSize
)i
, t
);
1992 emitir(IRTGI(IR_LE
), fr
, lj_ir_kint(J
, frofs
));
1995 for (i
= nvararg
; i
< nresults
; i
++)
1996 J
->base
[dst
+i
] = TREF_NIL
;
1997 if (nresults
!= 1 || dst
>= J
->maxslot
) {
1998 J
->maxslot
= dst
+ (BCReg
)nresults
;
2000 } else if (select_detect(J
)) { /* y = select(x, ...) */
2001 TRef tridx
= J
->base
[dst
-1];
2003 ptrdiff_t idx
= lj_ffrecord_select_mode(J
, tridx
, &J
->L
->base
[dst
-1]);
2004 if (idx
< 0) goto nyivarg
;
2005 if (idx
!= 0 && !tref_isinteger(tridx
)) {
2006 if (tref_isstr(tridx
))
2007 tridx
= emitir(IRTG(IR_STRTO
, IRT_NUM
), tridx
, 0);
2008 tridx
= emitir(IRTGI(IR_CONV
), tridx
, IRCONV_INT_NUM
|IRCONV_INDEX
);
2010 if (idx
!= 0 && tref_isk(tridx
)) {
2011 emitir(IRTGI(idx
<= nvararg
? IR_GE
: IR_LT
),
2012 fr
, lj_ir_kint(J
, frofs
+8*(int32_t)idx
));
2013 frofs
-= 8; /* Bias for 1-based index. */
2014 } else if (idx
<= nvararg
) { /* Compute size. */
2015 TRef tmp
= emitir(IRTI(IR_ADD
), fr
, lj_ir_kint(J
, -frofs
));
2017 emitir(IRTGI(IR_GE
), tmp
, lj_ir_kint(J
, 0));
2018 tr
= emitir(IRTI(IR_BSHR
), tmp
, lj_ir_kint(J
, 3));
2020 tridx
= emitir(IRTI(IR_ADD
), tridx
, lj_ir_kint(J
, -1));
2021 rec_idx_abc(J
, tr
, tridx
, (uint32_t)nvararg
);
2024 TRef tmp
= lj_ir_kint(J
, frofs
);
2026 TRef tmp2
= emitir(IRTI(IR_BSHL
), tridx
, lj_ir_kint(J
, 3));
2027 tmp
= emitir(IRTI(IR_ADD
), tmp2
, tmp
);
2029 tr
= lj_ir_kint(J
, 0);
2031 emitir(IRTGI(IR_LT
), fr
, tmp
);
2033 if (idx
!= 0 && idx
<= nvararg
) {
2035 TRef aref
, vbase
= emitir(IRT(IR_SUB
, IRT_IGC
), REF_BASE
, fr
);
2036 vbase
= emitir(IRT(IR_ADD
, IRT_PGC
), vbase
,
2037 lj_ir_kintpgc(J
, frofs
-(8<<LJ_FR2
)));
2038 t
= itype2irt(&J
->L
->base
[idx
-2-LJ_FR2
-nvararg
]);
2039 aref
= emitir(IRT(IR_AREF
, IRT_PGC
), vbase
, tridx
);
2040 tr
= lj_record_vload(J
, aref
, 0, t
);
2042 J
->base
[dst
-2-LJ_FR2
] = tr
;
2043 J
->maxslot
= dst
-1-LJ_FR2
;
2044 J
->bcskip
= 2; /* Skip CALLM + select. */
2047 setintV(&J
->errinfo
, BC_VARG
);
2048 lj_trace_err_info(J
, LJ_TRERR_NYIBC
);
2053 /* -- Record allocations -------------------------------------------------- */
2055 static TRef
rec_tnew(jit_State
*J
, uint32_t ah
)
2057 uint32_t asize
= ah
& 0x7ff;
2058 uint32_t hbits
= ah
>> 11;
2060 if (asize
== 0x7ff) asize
= 0x801;
2061 tr
= emitir(IRTG(IR_TNEW
, IRT_TAB
), asize
, hbits
);
2062 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2063 J
->rbchash
[(tr
& (RBCHASH_SLOTS
-1))].ref
= tref_ref(tr
);
2064 setmref(J
->rbchash
[(tr
& (RBCHASH_SLOTS
-1))].pc
, J
->pc
);
2065 setgcref(J
->rbchash
[(tr
& (RBCHASH_SLOTS
-1))].pt
, obj2gco(J
->pt
));
2070 /* -- Concatenation ------------------------------------------------------- */
2072 static TRef
rec_cat(jit_State
*J
, BCReg baseslot
, BCReg topslot
)
2074 TRef
*top
= &J
->base
[topslot
];
2075 TValue savetv
[5+LJ_FR2
];
2078 lj_assertJ(baseslot
< topslot
, "bad CAT arg");
2079 for (s
= baseslot
; s
<= topslot
; s
++)
2080 (void)getslot(J
, s
); /* Ensure all arguments have a reference. */
2081 if (tref_isnumber_str(top
[0]) && tref_isnumber_str(top
[-1])) {
2082 TRef tr
, hdr
, *trp
, *xbase
, *base
= &J
->base
[baseslot
];
2083 /* First convert numbers to strings. */
2084 for (trp
= top
; trp
>= base
; trp
--) {
2085 if (tref_isnumber(*trp
))
2086 *trp
= emitir(IRT(IR_TOSTR
, IRT_STR
), *trp
,
2087 tref_isnum(*trp
) ? IRTOSTR_NUM
: IRTOSTR_INT
);
2088 else if (!tref_isstr(*trp
))
2092 tr
= hdr
= emitir(IRT(IR_BUFHDR
, IRT_PGC
),
2093 lj_ir_kptr(J
, &J2G(J
)->tmpbuf
), IRBUFHDR_RESET
);
2095 tr
= emitir(IRTG(IR_BUFPUT
, IRT_PGC
), tr
, *trp
++);
2096 } while (trp
<= top
);
2097 tr
= emitir(IRTG(IR_BUFSTR
, IRT_STR
), tr
, hdr
);
2098 J
->maxslot
= (BCReg
)(xbase
- J
->base
);
2099 if (xbase
== base
) return tr
; /* Return simple concatenation result. */
2100 /* Pass partial result. */
2101 topslot
= J
->maxslot
--;
2104 setstrV(J
->L
, &ix
.keyv
, &J2G(J
)->strempty
); /* Simulate string result. */
2106 J
->maxslot
= topslot
-1;
2107 copyTV(J
->L
, &ix
.keyv
, &J
->L
->base
[topslot
]);
2109 copyTV(J
->L
, &ix
.tabv
, &J
->L
->base
[topslot
-1]);
2112 memcpy(savetv
, &J
->L
->base
[topslot
-1], sizeof(savetv
)); /* Save slots. */
2113 rec_mm_arith(J
, &ix
, MM_concat
); /* Call __concat metamethod. */
2114 memcpy(&J
->L
->base
[topslot
-1], savetv
, sizeof(savetv
)); /* Restore slots. */
2115 return 0; /* No result yet. */
2118 /* -- Record bytecode ops ------------------------------------------------- */
2120 /* Prepare for comparison. */
2121 static void rec_comp_prep(jit_State
*J
)
2123 /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
2124 if (J
->cur
.nsnap
== 1 && J
->cur
.snap
[0].ref
== J
->cur
.nins
)
2125 emitir_raw(IRT(IR_NOP
, IRT_NIL
), 0, 0);
2129 /* Fixup comparison. */
2130 static void rec_comp_fixup(jit_State
*J
, const BCIns
*pc
, int cond
)
2132 BCIns jmpins
= pc
[1];
2133 const BCIns
*npc
= pc
+ 2 + (cond
? bc_j(jmpins
) : 0);
2134 SnapShot
*snap
= &J
->cur
.snap
[J
->cur
.nsnap
-1];
2135 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
2137 SnapEntry
*flink
= &J
->cur
.snapmap
[snap
->mapofs
+ snap
->nent
];
2139 memcpy(&pcbase
, flink
, sizeof(uint64_t));
2140 pcbase
= (pcbase
& 0xff) | (u64ptr(npc
) << 8);
2141 memcpy(flink
, &pcbase
, sizeof(uint64_t));
2143 J
->cur
.snapmap
[snap
->mapofs
+ snap
->nent
] = SNAP_MKPC(npc
);
2146 if (bc_a(jmpins
) < J
->maxslot
) J
->maxslot
= bc_a(jmpins
);
2147 lj_snap_shrink(J
); /* Shrink last snapshot if possible. */
2150 /* Record the next bytecode instruction (_before_ it's executed). */
2151 void lj_record_ins(jit_State
*J
)
2160 /* Perform post-processing action before recording the next instruction. */
2161 if (LJ_UNLIKELY(J
->postproc
!= LJ_POST_NONE
)) {
2162 switch (J
->postproc
) {
2163 case LJ_POST_FIXCOMP
: /* Fixup comparison. */
2164 pc
= (const BCIns
*)(uintptr_t)J2G(J
)->tmptv
.u64
;
2165 rec_comp_fixup(J
, pc
, (!tvistruecond(&J2G(J
)->tmptv2
) ^ (bc_op(*pc
)&1)));
2167 case LJ_POST_FIXGUARD
: /* Fixup and emit pending guard. */
2168 case LJ_POST_FIXGUARDSNAP
: /* Fixup and emit pending guard and snapshot. */
2169 if (!tvistruecond(&J2G(J
)->tmptv2
)) {
2170 J
->fold
.ins
.o
^= 1; /* Flip guard to opposite. */
2171 if (J
->postproc
== LJ_POST_FIXGUARDSNAP
) {
2172 SnapShot
*snap
= &J
->cur
.snap
[J
->cur
.nsnap
-1];
2173 J
->cur
.snapmap
[snap
->mapofs
+snap
->nent
-1]--; /* False -> true. */
2176 lj_opt_fold(J
); /* Emit pending guard. */
2178 case LJ_POST_FIXBOOL
:
2179 if (!tvistruecond(&J2G(J
)->tmptv2
)) {
2181 TValue
*tv
= J
->L
->base
;
2182 for (s
= 0; s
< J
->maxslot
; s
++) /* Fixup stack slot (if any). */
2183 if (J
->base
[s
] == TREF_TRUE
&& tvisfalse(&tv
[s
])) {
2184 J
->base
[s
] = TREF_FALSE
;
2189 case LJ_POST_FIXCONST
:
2192 TValue
*tv
= J
->L
->base
;
2193 for (s
= 0; s
< J
->maxslot
; s
++) /* Constify stack slots (if any). */
2194 if (J
->base
[s
] == TREF_NIL
&& !tvisnil(&tv
[s
]))
2195 J
->base
[s
] = lj_record_constify(J
, &tv
[s
]);
2198 case LJ_POST_FFRETRY
: /* Suppress recording of retried fast function. */
2199 if (bc_op(*J
->pc
) >= BC__MAX
)
2202 default: lj_assertJ(0, "bad post-processing mode"); break;
2204 J
->postproc
= LJ_POST_NONE
;
2207 /* Need snapshot before recording next bytecode (e.g. after a store). */
2210 if (J
->pt
) lj_snap_purge(J
);
2215 /* Skip some bytecodes. */
2216 if (LJ_UNLIKELY(J
->bcskip
> 0)) {
2221 /* Record only closed loops for root traces. */
2223 if (J
->framedepth
== 0 &&
2224 (MSize
)((char *)pc
- (char *)J
->bc_min
) >= J
->bc_extent
)
2225 lj_trace_err(J
, LJ_TRERR_LLEAVE
);
2227 #ifdef LUA_USE_ASSERT
2233 rec_profile_ins(J
, pc
);
2236 /* Keep a copy of the runtime values of var/num/str operands. */
2237 #define rav (&ix.valv)
2238 #define rbv (&ix.tabv)
2239 #define rcv (&ix.keyv)
2246 switch (bcmode_a(op
)) {
2248 copyTV(J
->L
, rav
, &lbase
[ra
]); ix
.val
= ra
= getslot(J
, ra
); break;
2249 default: break; /* Handled later. */
2253 switch (bcmode_b(op
)) {
2254 case BCMnone
: rb
= 0; rc
= bc_d(ins
); break; /* Upgrade rc to 'rd'. */
2256 copyTV(J
->L
, rbv
, &lbase
[rb
]); ix
.tab
= rb
= getslot(J
, rb
); break;
2257 default: break; /* Handled later. */
2259 switch (bcmode_c(op
)) {
2261 copyTV(J
->L
, rcv
, &lbase
[rc
]); ix
.key
= rc
= getslot(J
, rc
); break;
2262 case BCMpri
: setpriV(rcv
, ~rc
); ix
.key
= rc
= TREF_PRI(IRT_NIL
+rc
); break;
2263 case BCMnum
: { cTValue
*tv
= proto_knumtv(J
->pt
, rc
);
2264 copyTV(J
->L
, rcv
, tv
); ix
.key
= rc
= tvisint(tv
) ? lj_ir_kint(J
, intV(tv
)) :
2265 tv
->u32
.hi
== LJ_KEYINDEX
? (lj_ir_kint(J
, 0) | TREF_KEYINDEX
) :
2266 lj_ir_knumint(J
, numV(tv
)); } break;
2267 case BCMstr
: { GCstr
*s
= gco2str(proto_kgc(J
->pt
, ~(ptrdiff_t)rc
));
2268 setstrV(J
->L
, rcv
, s
); ix
.key
= rc
= lj_ir_kstr(J
, s
); } break;
2269 default: break; /* Handled later. */
2274 /* -- Comparison ops ---------------------------------------------------- */
2276 case BC_ISLT
: case BC_ISGE
: case BC_ISLE
: case BC_ISGT
:
2278 if (tref_iscdata(ra
) || tref_iscdata(rc
)) {
2279 rec_mm_comp_cdata(J
, &ix
, op
, ((int)op
& 2) ? MM_le
: MM_lt
);
2283 /* Emit nothing for two numeric or string consts. */
2284 if (!(tref_isk2(ra
,rc
) && tref_isnumber_str(ra
) && tref_isnumber_str(rc
))) {
2285 IRType ta
= tref_isinteger(ra
) ? IRT_INT
: tref_type(ra
);
2286 IRType tc
= tref_isinteger(rc
) ? IRT_INT
: tref_type(rc
);
2289 /* Widen mixed number/int comparisons to number/number comparison. */
2290 if (ta
== IRT_INT
&& tc
== IRT_NUM
) {
2291 ra
= emitir(IRTN(IR_CONV
), ra
, IRCONV_NUM_INT
);
2293 } else if (ta
== IRT_NUM
&& tc
== IRT_INT
) {
2294 rc
= emitir(IRTN(IR_CONV
), rc
, IRCONV_NUM_INT
);
2296 ta
= IRT_NIL
; /* Force metamethod for different types. */
2297 } else if (!((ta
== IRT_FALSE
|| ta
== IRT_TRUE
) &&
2298 (tc
== IRT_FALSE
|| tc
== IRT_TRUE
))) {
2299 break; /* Interpreter will throw for two different types. */
2303 irop
= (int)op
- (int)BC_ISLT
+ (int)IR_LT
;
2304 if (ta
== IRT_NUM
) {
2305 if ((irop
& 1)) irop
^= 4; /* ISGE/ISGT are unordered. */
2306 if (!lj_ir_numcmp(numberVnum(rav
), numberVnum(rcv
), (IROp
)irop
))
2308 } else if (ta
== IRT_INT
) {
2309 if (!lj_ir_numcmp(numberVnum(rav
), numberVnum(rcv
), (IROp
)irop
))
2311 } else if (ta
== IRT_STR
) {
2312 if (!lj_ir_strcmp(strV(rav
), strV(rcv
), (IROp
)irop
)) irop
^= 1;
2313 ra
= lj_ir_call(J
, IRCALL_lj_str_cmp
, ra
, rc
);
2314 rc
= lj_ir_kint(J
, 0);
2317 rec_mm_comp(J
, &ix
, (int)op
);
2320 emitir(IRTG(irop
, ta
), ra
, rc
);
2321 rec_comp_fixup(J
, J
->pc
, ((int)op
^ irop
) & 1);
2325 case BC_ISEQV
: case BC_ISNEV
:
2326 case BC_ISEQS
: case BC_ISNES
:
2327 case BC_ISEQN
: case BC_ISNEN
:
2328 case BC_ISEQP
: case BC_ISNEP
:
2330 if (tref_iscdata(ra
) || tref_iscdata(rc
)) {
2331 rec_mm_comp_cdata(J
, &ix
, op
, MM_eq
);
2335 /* Emit nothing for two non-table, non-udata consts. */
2336 if (!(tref_isk2(ra
, rc
) && !(tref_istab(ra
) || tref_isudata(ra
)))) {
2339 diff
= lj_record_objcmp(J
, ra
, rc
, rav
, rcv
);
2340 if (diff
== 2 || !(tref_istab(ra
) || tref_isudata(ra
)))
2341 rec_comp_fixup(J
, J
->pc
, ((int)op
& 1) == !diff
);
2342 else if (diff
== 1) /* Only check __eq if different, but same type. */
2343 rec_mm_equal(J
, &ix
, (int)op
);
2347 /* -- Unary test and copy ops ------------------------------------------- */
2349 case BC_ISTC
: case BC_ISFC
:
2350 if ((op
& 1) == tref_istruecond(rc
))
2351 rc
= 0; /* Don't store if condition is not true. */
2353 case BC_IST
: case BC_ISF
: /* Type specialization suffices. */
2354 if (bc_a(pc
[1]) < J
->maxslot
)
2355 J
->maxslot
= bc_a(pc
[1]); /* Shrink used slots. */
2358 case BC_ISTYPE
: case BC_ISNUM
:
2359 /* These coercions need to correspond with lj_meta_istype(). */
2360 if (LJ_DUALNUM
&& rc
== ~LJ_TNUMX
+1)
2361 ra
= lj_opt_narrow_toint(J
, ra
);
2362 else if (rc
== ~LJ_TNUMX
+2)
2363 ra
= lj_ir_tonum(J
, ra
);
2364 else if (rc
== ~LJ_TSTR
+1)
2365 ra
= lj_ir_tostr(J
, ra
);
2366 /* else: type specialization suffices. */
2367 J
->base
[bc_a(ins
)] = ra
;
2370 /* -- Unary ops --------------------------------------------------------- */
2373 /* Type specialization already forces const result. */
2374 rc
= tref_istruecond(rc
) ? TREF_FALSE
: TREF_TRUE
;
2379 rc
= emitir(IRTI(IR_FLOAD
), rc
, IRFL_STR_LEN
);
2380 else if (!LJ_52
&& tref_istab(rc
))
2381 rc
= emitir(IRTI(IR_ALEN
), rc
, TREF_NIL
);
2383 rc
= rec_mm_len(J
, rc
, rcv
);
2386 /* -- Arithmetic ops ---------------------------------------------------- */
2389 if (tref_isnumber_str(rc
)) {
2390 rc
= lj_opt_narrow_unm(J
, rc
, rcv
);
2393 copyTV(J
->L
, &ix
.tabv
, rcv
);
2394 rc
= rec_mm_arith(J
, &ix
, MM_unm
);
2398 case BC_ADDNV
: case BC_SUBNV
: case BC_MULNV
: case BC_DIVNV
: case BC_MODNV
:
2399 /* Swap rb/rc and rbv/rcv. rav is temp. */
2400 ix
.tab
= rc
; ix
.key
= rc
= rb
; rb
= ix
.tab
;
2401 copyTV(J
->L
, rav
, rbv
);
2402 copyTV(J
->L
, rbv
, rcv
);
2403 copyTV(J
->L
, rcv
, rav
);
2407 case BC_ADDVN
: case BC_SUBVN
: case BC_MULVN
: case BC_DIVVN
:
2408 case BC_ADDVV
: case BC_SUBVV
: case BC_MULVV
: case BC_DIVVV
: {
2409 MMS mm
= bcmode_mm(op
);
2410 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
))
2411 rc
= lj_opt_narrow_arith(J
, rb
, rc
, rbv
, rcv
,
2412 (int)mm
- (int)MM_add
+ (int)IR_ADD
);
2414 rc
= rec_mm_arith(J
, &ix
, mm
);
2418 case BC_MODVN
: case BC_MODVV
:
2420 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
))
2421 rc
= lj_opt_narrow_mod(J
, rb
, rc
, rbv
, rcv
);
2423 rc
= rec_mm_arith(J
, &ix
, MM_mod
);
2427 if (tref_isnumber_str(rb
) && tref_isnumber_str(rc
))
2428 rc
= lj_opt_narrow_arith(J
, rb
, rc
, rbv
, rcv
, IR_POW
);
2430 rc
= rec_mm_arith(J
, &ix
, MM_pow
);
2433 /* -- Miscellaneous ops ------------------------------------------------- */
2436 rc
= rec_cat(J
, rb
, rc
);
2439 /* -- Constant and move ops --------------------------------------------- */
2442 /* Clear gap of method call to avoid resurrecting previous refs. */
2443 if (ra
> J
->maxslot
) {
2445 memset(J
->base
+ J
->maxslot
, 0, (ra
- J
->maxslot
) * sizeof(TRef
));
2451 case BC_KSTR
: case BC_KNUM
: case BC_KPRI
:
2454 rc
= lj_ir_kint(J
, (int32_t)(int16_t)rc
);
2457 if (LJ_FR2
&& ra
> J
->maxslot
)
2460 J
->base
[ra
++] = TREF_NIL
;
2461 if (rc
>= J
->maxslot
) J
->maxslot
= rc
+1;
2465 rc
= lj_ir_kgc(J
, proto_kgc(J
->pt
, ~(ptrdiff_t)rc
), IRT_CDATA
);
2469 /* -- Upvalue and function ops ------------------------------------------ */
2472 rc
= rec_upvalue(J
, rc
, 0);
2474 case BC_USETV
: case BC_USETS
: case BC_USETN
: case BC_USETP
:
2475 rec_upvalue(J
, ra
, rc
);
2478 /* -- Table ops --------------------------------------------------------- */
2480 case BC_GGET
: case BC_GSET
:
2481 settabV(J
->L
, &ix
.tabv
, tabref(J
->fn
->l
.env
));
2482 ix
.tab
= emitir(IRT(IR_FLOAD
, IRT_TAB
), getcurrf(J
), IRFL_FUNC_ENV
);
2483 ix
.idxchain
= LJ_MAX_IDXCHAIN
;
2484 rc
= lj_record_idx(J
, &ix
);
2487 case BC_TGETB
: case BC_TSETB
:
2488 setintV(&ix
.keyv
, (int32_t)rc
);
2489 ix
.key
= lj_ir_kint(J
, (int32_t)rc
);
2491 case BC_TGETV
: case BC_TGETS
: case BC_TSETV
: case BC_TSETS
:
2492 ix
.idxchain
= LJ_MAX_IDXCHAIN
;
2493 rc
= lj_record_idx(J
, &ix
);
2495 case BC_TGETR
: case BC_TSETR
:
2497 rc
= lj_record_idx(J
, &ix
);
2501 rec_tsetm(J
, ra
, (BCReg
)(J
->L
->top
- J
->L
->base
), (int32_t)rcv
->u32
.lo
);
2502 J
->maxslot
= ra
; /* The table slot at ra-1 is the highest used slot. */
2506 rc
= rec_tnew(J
, rc
);
2509 rc
= emitir(IRTG(IR_TDUP
, IRT_TAB
),
2510 lj_ir_ktab(J
, gco2tab(proto_kgc(J
->pt
, ~(ptrdiff_t)rc
))), 0);
2511 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2512 J
->rbchash
[(rc
& (RBCHASH_SLOTS
-1))].ref
= tref_ref(rc
);
2513 setmref(J
->rbchash
[(rc
& (RBCHASH_SLOTS
-1))].pc
, pc
);
2514 setgcref(J
->rbchash
[(rc
& (RBCHASH_SLOTS
-1))].pt
, obj2gco(J
->pt
));
2518 /* -- Calls and vararg handling ----------------------------------------- */
2521 J
->base
[ra
] = getslot(J
, ra
-3);
2522 J
->base
[ra
+1+LJ_FR2
] = getslot(J
, ra
-2);
2523 J
->base
[ra
+2+LJ_FR2
] = getslot(J
, ra
-1);
2524 { /* Do the actual copy now because lj_record_call needs the values. */
2525 TValue
*b
= &J
->L
->base
[ra
];
2526 copyTV(J
->L
, b
, b
-3);
2527 copyTV(J
->L
, b
+1+LJ_FR2
, b
-2);
2528 copyTV(J
->L
, b
+2+LJ_FR2
, b
-1);
2530 lj_record_call(J
, ra
, (ptrdiff_t)rc
-1);
2533 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
2535 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
- LJ_FR2
;
2538 lj_record_call(J
, ra
, (ptrdiff_t)rc
-1);
2542 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
- LJ_FR2
;
2545 lj_record_tailcall(J
, ra
, (ptrdiff_t)rc
-1);
2549 rec_varg(J
, ra
, (ptrdiff_t)rb
-1);
2552 /* -- Returns ----------------------------------------------------------- */
2555 /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
2556 rc
= (BCReg
)(J
->L
->top
- J
->L
->base
) - ra
+ 1;
2558 case BC_RET
: case BC_RET0
: case BC_RET1
:
2562 lj_record_ret(J
, ra
, (ptrdiff_t)rc
-1);
2565 /* -- Loops and branches ------------------------------------------------ */
2568 if (rec_for(J
, pc
, 0) != LOOPEV_LEAVE
)
2569 J
->loopref
= J
->cur
.nins
;
2572 lj_assertJ(bc_op(pc
[(ptrdiff_t)rc
-BCBIAS_J
]) == BC_JFORL
,
2573 "JFORI does not point to JFORL");
2574 if (rec_for(J
, pc
, 0) != LOOPEV_LEAVE
) /* Link to existing loop. */
2575 lj_record_stop(J
, LJ_TRLINK_ROOT
, bc_d(pc
[(ptrdiff_t)rc
-BCBIAS_J
]));
2576 /* Continue tracing if the loop is not entered. */
2580 rec_loop_interp(J
, pc
, rec_for(J
, pc
+((ptrdiff_t)rc
-BCBIAS_J
), 1));
2583 rec_loop_interp(J
, pc
, rec_iterl(J
, *pc
));
2586 rec_loop_interp(J
, pc
, rec_itern(J
, ra
, rb
));
2589 rec_loop_interp(J
, pc
, rec_loop(J
, ra
, 1));
2593 rec_loop_jit(J
, rc
, rec_for(J
, pc
+bc_j(traceref(J
, rc
)->startins
), 1));
2596 rec_loop_jit(J
, rc
, rec_iterl(J
, traceref(J
, rc
)->startins
));
2599 rec_loop_jit(J
, rc
, rec_loop(J
, ra
,
2600 !bc_isret(bc_op(traceref(J
, rc
)->startins
)) &&
2601 bc_op(traceref(J
, rc
)->startins
) != BC_ITERN
));
2609 lj_trace_err(J
, LJ_TRERR_BLACKL
);
2613 if (ra
< J
->maxslot
)
2614 J
->maxslot
= ra
; /* Shrink used slots. */
2621 /* -- Function headers -------------------------------------------------- */
2627 rec_func_jit(J
, rc
);
2635 /* Cannot happen. No hotcall counting for varag funcs. */
2636 lj_assertJ(0, "unsupported vararg hotcall");
2641 lj_ffrecord_func(J
);
2645 if (op
>= BC__MAX
) {
2646 lj_ffrecord_func(J
);
2652 setintV(&J
->errinfo
, (int32_t)op
);
2653 lj_trace_err_info(J
, LJ_TRERR_NYIBC
);
2657 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2658 if (bcmode_a(op
) == BCMdst
&& rc
) {
2660 if (ra
>= J
->maxslot
) {
2662 if (ra
> J
->maxslot
) J
->base
[ra
-1] = 0;
2672 /* Limit the number of recorded IR instructions and constants. */
2673 if (J
->cur
.nins
> REF_FIRST
+(IRRef
)J
->param
[JIT_P_maxrecord
] ||
2674 J
->cur
.nk
< REF_BIAS
-(IRRef
)J
->param
[JIT_P_maxirconst
])
2675 lj_trace_err(J
, LJ_TRERR_TRACEOV
);
2678 /* -- Recording setup ----------------------------------------------------- */
2680 /* Setup recording for a root trace started by a hot loop. */
2681 static const BCIns
*rec_setup_root(jit_State
*J
)
2683 /* Determine the next PC and the bytecode range for the loop. */
2684 const BCIns
*pcj
, *pc
= J
->pc
;
2686 BCReg ra
= bc_a(ins
);
2687 switch (bc_op(ins
)) {
2689 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
2694 if (bc_op(pc
[-1]) == BC_JLOOP
)
2695 lj_trace_err(J
, LJ_TRERR_LINNER
);
2696 lj_assertJ(bc_op(pc
[-1]) == BC_ITERC
, "no ITERC before ITERL");
2697 J
->maxslot
= ra
+ bc_b(pc
[-1]) - 1;
2698 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
2700 lj_assertJ(bc_op(pc
[-1]) == BC_JMP
, "ITERL does not point to JMP+1");
2704 lj_assertJ(bc_op(pc
[1]) == BC_ITERL
, "no ITERL after ITERN");
2706 J
->bc_extent
= (MSize
)(-bc_j(pc
[1]))*sizeof(BCIns
);
2707 J
->bc_min
= pc
+2 + bc_j(pc
[1]);
2708 J
->state
= LJ_TRACE_RECORD_1ST
; /* Record the first ITERN, too. */
2711 /* Only check BC range for real loops, but not for "repeat until true". */
2712 pcj
= pc
+ bc_j(ins
);
2714 if (bc_op(ins
) == BC_JMP
&& bc_j(ins
) < 0) {
2715 J
->bc_min
= pcj
+1 + bc_j(ins
);
2716 J
->bc_extent
= (MSize
)(-bc_j(ins
))*sizeof(BCIns
);
2724 /* No bytecode range check for down-recursive root traces. */
2725 J
->maxslot
= ra
+ bc_d(ins
) - 1;
2728 /* No bytecode range check for root traces started by a hot call. */
2729 J
->maxslot
= J
->pt
->numparams
;
2735 /* No bytecode range check for stitched traces. */
2739 lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins
));
2745 /* Setup for recording a new trace. */
2746 void lj_record_setup(jit_State
*J
)
2750 /* Initialize state related to current trace. */
2751 memset(J
->slot
, 0, sizeof(J
->slot
));
2752 memset(J
->chain
, 0, sizeof(J
->chain
));
2753 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2754 memset(J
->rbchash
, 0, sizeof(J
->rbchash
));
2756 memset(J
->bpropcache
, 0, sizeof(J
->bpropcache
));
2757 J
->scev
.idx
= REF_NIL
;
2758 setmref(J
->scev
.pc
, NULL
);
2760 J
->baseslot
= 1+LJ_FR2
; /* Invoking function is at base[-1-LJ_FR2]. */
2761 J
->base
= J
->slot
+ J
->baseslot
;
2766 J
->instunroll
= J
->param
[JIT_P_instunroll
];
2767 J
->loopunroll
= J
->param
[JIT_P_loopunroll
];
2771 J
->bc_min
= NULL
; /* Means no limit. */
2772 J
->bc_extent
= ~(MSize
)0;
2774 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2775 emitir_raw(IRT(IR_BASE
, IRT_PGC
), J
->parent
, J
->exitno
);
2776 for (i
= 0; i
<= 2; i
++) {
2777 IRIns
*ir
= IR(REF_NIL
-i
);
2779 ir
->t
.irt
= (uint8_t)(IRT_NIL
+i
);
2783 J
->cur
.nk
= REF_TRUE
;
2786 setmref(J
->cur
.startpc
, J
->pc
);
2787 if (J
->parent
) { /* Side trace. */
2788 GCtrace
*T
= traceref(J
, J
->parent
);
2789 TraceNo root
= T
->root
? T
->root
: J
->parent
;
2790 J
->cur
.root
= (uint16_t)root
;
2791 J
->cur
.startins
= BCINS_AD(BC_JMP
, 0, 0);
2792 /* Check whether we could at least potentially form an extra loop. */
2793 if (J
->exitno
== 0 && T
->snap
[0].nent
== 0) {
2794 /* We can narrow a FORL for some side traces, too. */
2795 if (J
->pc
> proto_bc(J
->pt
) && bc_op(J
->pc
[-1]) == BC_JFORI
&&
2796 bc_d(J
->pc
[bc_j(J
->pc
[-1])-1]) == root
) {
2798 rec_for_loop(J
, J
->pc
-1, &J
->scev
, 1);
2802 J
->startpc
= NULL
; /* Prevent forming an extra loop. */
2804 lj_snap_replay(J
, T
);
2806 if ((traceref(J
, J
->cur
.root
)->nchild
>= J
->param
[JIT_P_maxside
] ||
2807 T
->snap
[J
->exitno
].count
>= J
->param
[JIT_P_hotexit
] +
2808 J
->param
[JIT_P_tryside
])) {
2809 if (bc_op(*J
->pc
) == BC_JLOOP
) {
2810 BCIns startins
= traceref(J
, bc_d(*J
->pc
))->startins
;
2811 if (bc_op(startins
) == BC_ITERN
)
2812 rec_itern(J
, bc_a(startins
), bc_b(startins
));
2814 lj_record_stop(J
, LJ_TRLINK_INTERP
, 0);
2816 } else { /* Root trace. */
2818 J
->cur
.startins
= *J
->pc
;
2819 J
->pc
= rec_setup_root(J
);
2820 /* Note: the loop instruction itself is recorded at the end and not
2821 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2822 ** The one exception is BC_ITERN, which sets LJ_TRACE_RECORD_1ST.
2825 if (bc_op(J
->cur
.startins
) == BC_FORL
)
2826 rec_for_loop(J
, J
->pc
-1, &J
->scev
, 1);
2827 else if (bc_op(J
->cur
.startins
) == BC_ITERC
)
2829 if (1 + J
->pt
->framesize
>= LJ_MAX_JSLOTS
)
2830 lj_trace_err(J
, LJ_TRERR_STACKOV
);
2836 #ifdef LUAJIT_ENABLE_CHECKHOOK
2837 /* Regularly check for instruction/line hooks from compiled code and
2838 ** exit to the interpreter if the hooks are set.
2840 ** This is a compile-time option and disabled by default, since the
2841 ** hook checks may be quite expensive in tight loops.
2843 ** Note this is only useful if hooks are *not* set most of the time.
2844 ** Use this only if you want to *asynchronously* interrupt the execution.
2846 ** You can set the instruction hook via lua_sethook() with a count of 1
2847 ** from a signal handler or another native thread. Please have a look
2848 ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2851 TRef tr
= emitir(IRT(IR_XLOAD
, IRT_U8
),
2852 lj_ir_kptr(J
, &J2G(J
)->hookmask
), IRXLOAD_VOLATILE
);
2853 tr
= emitir(IRTI(IR_BAND
), tr
, lj_ir_kint(J
, (LUA_MASKLINE
|LUA_MASKCOUNT
)));
2854 emitir(IRTGI(IR_EQ
), tr
, lj_ir_kint(J
, 0));