2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate a register or RID_ZERO. */
22 static Reg
ra_alloc1z(ASMState
*as
, IRRef ref
, RegSet allow
)
26 if (!(allow
& RSET_FPR
) && irref_isk(ref
) && IR(ref
)->i
== 0)
28 r
= ra_allocref(as
, ref
, allow
);
35 /* Allocate two source registers for three-operand instructions. */
36 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
38 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
39 Reg left
= irl
->r
, right
= irr
->r
;
40 if (ra_hasreg(left
)) {
43 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
46 } else if (ra_hasreg(right
)) {
48 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
49 } else if (ra_hashint(right
)) {
50 right
= ra_alloc1z(as
, ir
->op2
, allow
);
51 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
53 left
= ra_alloc1z(as
, ir
->op1
, allow
);
54 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
56 return left
| (right
<< 8);
59 /* -- Guard handling ------------------------------------------------------ */
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
64 /* Setup spare long-range jump slots per mcarea. */
65 static void asm_sparejump_setup(ASMState
*as
)
67 MCode
*mxp
= as
->mcbot
;
68 /* Assumes sizeof(MCLink) == 8. */
69 if (((uintptr_t)mxp
& (LJ_PAGESIZE
-1)) == 8) {
70 lua_assert(MIPSI_NOP
== 0);
71 memset(mxp
+2, 0, MIPS_SPAREJUMP
*8);
72 mxp
+= MIPS_SPAREJUMP
*2;
73 lua_assert(mxp
< as
->mctop
);
74 lj_mcode_commitbot(as
->J
, mxp
);
76 as
->mclim
= as
->mcbot
+ MCLIM_REDZONE
;
80 /* Setup exit stub after the end of each trace. */
81 static void asm_exitstub_setup(ASMState
*as
)
83 MCode
*mxp
= as
->mctop
;
84 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
85 *--mxp
= MIPSI_LI
|MIPSF_T(RID_TMP
)|as
->T
->traceno
;
86 *--mxp
= MIPSI_J
|((((uintptr_t)(void *)lj_vm_exit_handler
)>>2)&0x03ffffffu
);
87 lua_assert(((uintptr_t)mxp
^ (uintptr_t)(void *)lj_vm_exit_handler
)>>28 == 0);
88 *--mxp
= MIPSI_SW
|MIPSF_T(RID_TMP
)|MIPSF_S(RID_SP
)|0;
92 /* Keep this in-sync with exitstub_trace_addr(). */
93 #define asm_exitstub_addr(as) ((as)->mctop)
95 /* Emit conditional branch to exit for guard. */
96 static void asm_guard(ASMState
*as
, MIPSIns mi
, Reg rs
, Reg rt
)
98 MCode
*target
= asm_exitstub_addr(as
);
100 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
104 mi
= mi
^ ((mi
>>28) == 1 ? 0x04000000u
: 0x00010000u
); /* Invert cond. */
105 target
= p
; /* Patch target later in asm_loop_fixup. */
107 emit_ti(as
, MIPSI_LI
, RID_TMP
, as
->snapno
);
108 emit_branch(as
, mi
, rs
, rt
, target
);
111 /* -- Operand fusion ------------------------------------------------------ */
113 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
114 #define CONFLICT_SEARCH_LIM 31
116 /* Check if there's no conflicting instruction between curins and ref. */
117 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
120 IRRef i
= as
->curins
;
121 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
122 return 0; /* Give up, ref is too far away. */
124 if (ir
[i
].o
== conflict
)
125 return 0; /* Conflict found. */
126 return 1; /* Ok, no conflict. */
129 /* Fuse the array base of colocated arrays. */
130 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
133 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
134 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
135 return (int32_t)sizeof(GCtab
);
139 /* Fuse array/hash/upvalue reference into register+offset operand. */
140 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
143 if (ra_noreg(ir
->r
)) {
144 if (ir
->o
== IR_AREF
) {
145 if (mayfuse(as
, ref
)) {
146 if (irref_isk(ir
->op2
)) {
147 IRRef tab
= IR(ir
->op1
)->op1
;
148 int32_t ofs
= asm_fuseabase(as
, tab
);
149 IRRef refa
= ofs
? tab
: ir
->op1
;
150 ofs
+= 8*IR(ir
->op2
)->i
;
153 return ra_alloc1(as
, refa
, allow
);
157 } else if (ir
->o
== IR_HREFK
) {
158 if (mayfuse(as
, ref
)) {
159 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
162 return ra_alloc1(as
, ir
->op1
, allow
);
165 } else if (ir
->o
== IR_UREFC
) {
166 if (irref_isk(ir
->op1
)) {
167 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
168 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
169 int32_t jgl
= (intptr_t)J2G(as
->J
);
170 if ((uint32_t)(ofs
-jgl
) < 65536) {
171 *ofsp
= ofs
-jgl
-32768;
174 *ofsp
= (int16_t)ofs
;
175 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
181 return ra_alloc1(as
, ref
, allow
);
184 /* Fuse XLOAD/XSTORE reference into load/store operand. */
185 static void asm_fusexref(ASMState
*as
, MIPSIns mi
, Reg rt
, IRRef ref
,
186 RegSet allow
, int32_t ofs
)
190 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
191 if (ir
->o
== IR_ADD
) {
193 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
197 } else if (ir
->o
== IR_STRREF
) {
198 int32_t ofs2
= 65536;
199 lua_assert(ofs
== 0);
200 ofs
= (int32_t)sizeof(GCstr
);
201 if (irref_isk(ir
->op2
)) {
202 ofs2
= ofs
+ IR(ir
->op2
)->i
;
204 } else if (irref_isk(ir
->op1
)) {
205 ofs2
= ofs
+ IR(ir
->op1
)->i
;
208 if (!checki16(ofs2
)) {
209 /* NYI: Fuse ADD with constant. */
210 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
211 right
= (left
>> 8); left
&= 255;
212 emit_hsi(as
, mi
, rt
, RID_TMP
, ofs
);
213 emit_dst(as
, MIPSI_ADDU
, RID_TMP
, left
, right
);
219 base
= ra_alloc1(as
, ref
, allow
);
220 emit_hsi(as
, mi
, rt
, base
, ofs
);
223 /* -- Calls --------------------------------------------------------------- */
225 /* Generate a call to a C function. */
226 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
228 uint32_t n
, nargs
= CCI_NARGS(ci
);
230 Reg gpr
, fpr
= REGARG_FIRSTFPR
;
231 if ((void *)ci
->func
)
232 emit_call(as
, (void *)ci
->func
);
233 for (gpr
= REGARG_FIRSTGPR
; gpr
<= REGARG_LASTGPR
; gpr
++)
234 as
->cost
[gpr
] = REGCOST(~0u, ASMREF_L
);
235 gpr
= REGARG_FIRSTGPR
;
236 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
240 if (irt_isfp(ir
->t
) && fpr
<= REGARG_LASTFPR
&&
241 !(ci
->flags
& CCI_VARARG
)) {
242 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
243 ra_leftov(as
, fpr
, ref
);
245 gpr
+= irt_isnum(ir
->t
) ? 2 : 1;
247 fpr
= REGARG_LASTFPR
+1;
248 if (irt_isnum(ir
->t
)) gpr
= (gpr
+1) & ~1;
249 if (gpr
<= REGARG_LASTGPR
) {
250 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
251 if (irt_isfp(ir
->t
)) {
252 RegSet of
= as
->freeset
;
254 /* Workaround to protect argument GPRs from being used for remat. */
255 as
->freeset
&= ~RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1);
256 r
= ra_alloc1(as
, ref
, RSET_FPR
);
257 as
->freeset
|= (of
& RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1));
258 if (irt_isnum(ir
->t
)) {
259 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?0:1), r
+1);
260 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?1:0), r
);
261 lua_assert(rset_test(as
->freeset
, gpr
+1)); /* Already evicted. */
263 } else if (irt_isfloat(ir
->t
)) {
264 emit_tg(as
, MIPSI_MFC1
, gpr
, r
);
268 ra_leftov(as
, gpr
, ref
);
272 Reg r
= ra_alloc1z(as
, ref
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
273 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
274 emit_spstore(as
, ir
, r
, ofs
);
275 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
279 fpr
= REGARG_LASTFPR
+1;
280 if (gpr
<= REGARG_LASTGPR
)
288 /* Setup result reg/sp for call. Evict scratch regs. */
289 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
291 RegSet drop
= RSET_SCRATCH
;
292 int hiop
= ((ir
+1)->o
== IR_HIOP
);
293 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
295 if (ra_hasreg(ir
->r
))
296 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
297 if (hiop
&& ra_hasreg((ir
+1)->r
))
298 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
299 ra_evictset(as
, drop
); /* Evictions must be performed first. */
301 lua_assert(!irt_ispri(ir
->t
));
302 if (irt_isfp(ir
->t
)) {
303 if ((ci
->flags
& CCI_CASTU64
)) {
304 int32_t ofs
= sps_scale(ir
->s
);
306 if (ra_hasreg(dest
)) {
308 ra_modified(as
, dest
);
309 emit_tg(as
, MIPSI_MTC1
, RID_RETHI
, dest
+1);
310 emit_tg(as
, MIPSI_MTC1
, RID_RETLO
, dest
);
313 emit_tsi(as
, MIPSI_SW
, RID_RETLO
, RID_SP
, ofs
+(LJ_BE
?4:0));
314 emit_tsi(as
, MIPSI_SW
, RID_RETHI
, RID_SP
, ofs
+(LJ_BE
?0:4));
317 ra_destreg(as
, ir
, RID_FPRET
);
322 ra_destreg(as
, ir
, RID_RET
);
327 static void asm_call(ASMState
*as
, IRIns
*ir
)
329 IRRef args
[CCI_NARGS_MAX
];
330 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
331 asm_collectargs(as
, ir
, ci
, args
);
332 asm_setupresult(as
, ir
, ci
);
333 asm_gencall(as
, ci
, args
);
336 static void asm_callx(ASMState
*as
, IRIns
*ir
)
338 IRRef args
[CCI_NARGS_MAX
];
342 ci
.flags
= asm_callx_flags(as
, ir
);
343 asm_collectargs(as
, ir
, &ci
, args
);
344 asm_setupresult(as
, ir
, &ci
);
345 func
= ir
->op2
; irf
= IR(func
);
346 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
347 if (irref_isk(func
)) { /* Call to constant address. */
348 ci
.func
= (ASMFunction
)(void *)(irf
->i
);
349 } else { /* Need specific register for indirect calls. */
350 Reg r
= ra_alloc1(as
, func
, RID2RSET(RID_CFUNCADDR
));
352 if (r
== RID_CFUNCADDR
)
355 *--p
= MIPSI_MOVE
| MIPSF_D(RID_CFUNCADDR
) | MIPSF_S(r
);
356 *--p
= MIPSI_JALR
| MIPSF_S(r
);
358 ci
.func
= (ASMFunction
)(void *)0;
360 asm_gencall(as
, &ci
, args
);
363 static void asm_callid(ASMState
*as
, IRIns
*ir
, IRCallID id
)
365 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
369 asm_setupresult(as
, ir
, ci
);
370 asm_gencall(as
, ci
, args
);
373 static void asm_callround(ASMState
*as
, IRIns
*ir
, IRCallID id
)
375 /* The modified regs must match with the *.dasc implementation. */
376 RegSet drop
= RID2RSET(RID_R1
)|RID2RSET(RID_R12
)|RID2RSET(RID_FPRET
)|
377 RID2RSET(RID_F2
)|RID2RSET(RID_F4
)|RID2RSET(REGARG_FIRSTFPR
);
378 if (ra_hasreg(ir
->r
)) rset_clear(drop
, ir
->r
);
379 ra_evictset(as
, drop
);
380 ra_destreg(as
, ir
, RID_FPRET
);
381 emit_call(as
, (void *)lj_ir_callinfo
[id
].func
);
382 ra_leftov(as
, REGARG_FIRSTFPR
, ir
->op1
);
385 /* -- Returns ------------------------------------------------------------- */
387 /* Return to lower frame. Guard that it goes to the right spot. */
388 static void asm_retf(ASMState
*as
, IRIns
*ir
)
390 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
391 void *pc
= ir_kptr(IR(ir
->op2
));
392 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
393 as
->topslot
-= (BCReg
)delta
;
394 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
395 emit_setgl(as
, base
, jit_base
);
396 emit_addptr(as
, base
, -8*delta
);
397 asm_guard(as
, MIPSI_BNE
, RID_TMP
,
398 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
399 emit_tsi(as
, MIPSI_LW
, RID_TMP
, base
, -8);
402 /* -- Type conversions ---------------------------------------------------- */
404 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
406 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
407 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
408 asm_guard(as
, MIPSI_BC1F
, 0, 0);
409 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmp
, left
);
410 emit_fg(as
, MIPSI_CVT_D_W
, tmp
, tmp
);
411 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
412 emit_fg(as
, MIPSI_CVT_W_D
, tmp
, left
);
415 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
417 RegSet allow
= RSET_FPR
;
418 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
419 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
420 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
421 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
422 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
423 emit_fgh(as
, MIPSI_ADD_D
, tmp
, left
, right
);
426 static void asm_conv(ASMState
*as
, IRIns
*ir
)
428 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
429 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
430 IRRef lref
= ir
->op1
;
431 lua_assert(irt_type(ir
->t
) != st
);
432 lua_assert(!(irt_isint64(ir
->t
) ||
433 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
434 if (irt_isfp(ir
->t
)) {
435 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
436 if (stfp
) { /* FP to FP conversion. */
437 emit_fg(as
, st
== IRT_NUM
? MIPSI_CVT_S_D
: MIPSI_CVT_D_S
,
438 dest
, ra_alloc1(as
, lref
, RSET_FPR
));
439 } else if (st
== IRT_U32
) { /* U32 to FP conversion. */
440 /* y = (x ^ 0x8000000) + 2147483648.0 */
441 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
442 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
443 emit_fgh(as
, irt_isfloat(ir
->t
) ? MIPSI_ADD_S
: MIPSI_ADD_D
,
445 emit_fg(as
, irt_isfloat(ir
->t
) ? MIPSI_CVT_S_W
: MIPSI_CVT_D_W
,
447 if (irt_isfloat(ir
->t
))
448 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
449 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,4f000000
)),
452 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
453 (void *)lj_ir_k64_find(as
->J
, U64x(41e00000
,00000000)),
455 emit_tg(as
, MIPSI_MTC1
, RID_TMP
, dest
);
456 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, left
);
457 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
458 } else { /* Integer to FP conversion. */
459 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
460 emit_fg(as
, irt_isfloat(ir
->t
) ? MIPSI_CVT_S_W
: MIPSI_CVT_D_W
,
462 emit_tg(as
, MIPSI_MTC1
, left
, dest
);
464 } else if (stfp
) { /* FP to integer conversion. */
465 if (irt_isguard(ir
->t
)) {
466 /* Checked conversions are only supported from number to int. */
467 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
468 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
470 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
471 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
472 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
473 if (irt_isu32(ir
->t
)) {
474 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
475 emit_dst(as
, MIPSI_XOR
, dest
, dest
, RID_TMP
);
476 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
477 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
478 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_FLOOR_W_S
: MIPSI_FLOOR_W_D
,
480 emit_fgh(as
, st
== IRT_FLOAT
? MIPSI_SUB_S
: MIPSI_SUB_D
,
483 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
484 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,4f000000
)),
487 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
488 (void *)lj_ir_k64_find(as
->J
, U64x(41e00000
,00000000)),
491 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
492 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_TRUNC_W_S
: MIPSI_TRUNC_W_D
,
497 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
498 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
499 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
500 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
501 if ((ir
->op2
& IRCONV_SEXT
)) {
502 if ((as
->flags
& JIT_F_MIPS32R2
)) {
503 emit_dst(as
, st
== IRT_I8
? MIPSI_SEB
: MIPSI_SEH
, dest
, 0, left
);
505 uint32_t shift
= st
== IRT_I8
? 24 : 16;
506 emit_dta(as
, MIPSI_SRA
, dest
, dest
, shift
);
507 emit_dta(as
, MIPSI_SLL
, dest
, left
, shift
);
510 emit_tsi(as
, MIPSI_ANDI
, dest
, left
,
511 (int32_t)(st
== IRT_U8
? 0xff : 0xffff));
513 } else { /* 32/64 bit integer conversions. */
514 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
515 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
521 static void asm_conv64(ASMState
*as
, IRIns
*ir
)
523 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
524 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
528 args
[LJ_BE
?0:1] = ir
->op1
;
529 args
[LJ_BE
?1:0] = (ir
-1)->op1
;
530 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) {
531 id
= IRCALL_fp64_d2l
+ ((st
== IRT_FLOAT
) ? 2 : 0) + (dt
- IRT_I64
);
534 id
= IRCALL_fp64_l2d
+ ((dt
== IRT_FLOAT
) ? 2 : 0) + (st
- IRT_I64
);
536 ci
= &lj_ir_callinfo
[id
];
537 asm_setupresult(as
, ir
, ci
);
538 asm_gencall(as
, ci
, args
);
542 static void asm_strto(ASMState
*as
, IRIns
*ir
)
544 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
546 RegSet drop
= RSET_SCRATCH
;
547 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
548 ra_evictset(as
, drop
);
549 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_ZERO
); /* Test return status. */
550 args
[0] = ir
->op1
; /* GCstr *str */
551 args
[1] = ASMREF_TMP1
; /* TValue *n */
552 asm_gencall(as
, ci
, args
);
553 /* Store the result to the spill slot or temp slots. */
554 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
),
555 RID_SP
, sps_scale(ir
->s
));
558 /* Get pointer to TValue. */
559 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
562 if (irt_isnum(ir
->t
)) {
563 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
564 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
565 else /* Otherwise force a spill and use the spill slot. */
566 emit_tsi(as
, MIPSI_ADDIU
, dest
, RID_SP
, ra_spill(as
, ir
));
568 /* Otherwise use g->tmptv to hold the TValue. */
569 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
571 emit_tsi(as
, MIPSI_ADDIU
, dest
, RID_JGL
, offsetof(global_State
, tmptv
)-32768);
572 if (!irt_ispri(ir
->t
)) {
573 Reg src
= ra_alloc1(as
, ref
, allow
);
574 emit_setgl(as
, src
, tmptv
.gcr
);
576 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
577 emit_setgl(as
, type
, tmptv
.it
);
581 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
586 if (irt_isnum(IR(ir
->op1
)->t
) || (ir
+1)->o
== IR_HIOP
) {
587 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
588 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
589 asm_setupresult(as
, ir
, ci
); /* GCstr * */
590 asm_gencall(as
, ci
, args
);
591 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op1
);
593 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
594 args
[1] = ir
->op1
; /* int32_t k */
595 asm_setupresult(as
, ir
, ci
); /* GCstr * */
596 asm_gencall(as
, ci
, args
);
600 /* -- Memory references --------------------------------------------------- */
602 static void asm_aref(ASMState
*as
, IRIns
*ir
)
604 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
606 if (irref_isk(ir
->op2
)) {
607 IRRef tab
= IR(ir
->op1
)->op1
;
608 int32_t ofs
= asm_fuseabase(as
, tab
);
609 IRRef refa
= ofs
? tab
: ir
->op1
;
610 ofs
+= 8*IR(ir
->op2
)->i
;
612 base
= ra_alloc1(as
, refa
, RSET_GPR
);
613 emit_tsi(as
, MIPSI_ADDIU
, dest
, base
, ofs
);
617 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
618 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
619 emit_dst(as
, MIPSI_ADDU
, dest
, RID_TMP
, base
);
620 emit_dta(as
, MIPSI_SLL
, RID_TMP
, idx
, 3);
623 /* Inlined hash lookup. Specialized for key type and for const keys.
624 ** The equivalent C code is:
625 ** Node *n = hashkey(t, key);
627 ** if (lj_obj_equal(&n->key, key)) return &n->val;
628 ** } while ((n = nextnode(n)));
631 static void asm_href(ASMState
*as
, IRIns
*ir
)
633 RegSet allow
= RSET_GPR
;
634 int destused
= ra_used(ir
);
635 Reg dest
= ra_dest(as
, ir
, allow
);
636 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
637 Reg key
= RID_NONE
, type
= RID_NONE
, tmpnum
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
638 IRRef refkey
= ir
->op2
;
639 IRIns
*irkey
= IR(refkey
);
640 IRType1 kt
= irkey
->t
;
642 MCLabel l_end
, l_loop
, l_next
;
644 rset_clear(allow
, tab
);
646 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
647 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
648 } else if (!irt_ispri(kt
)) {
649 key
= ra_alloc1(as
, refkey
, allow
);
650 rset_clear(allow
, key
);
651 type
= ra_allock(as
, irt_toitype(irkey
->t
), allow
);
652 rset_clear(allow
, type
);
654 tmp2
= ra_scratch(as
, allow
);
655 rset_clear(allow
, tmp2
);
657 /* Key not found in chain: load niltv. */
658 l_end
= emit_label(as
);
660 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
662 *--as
->mcp
= MIPSI_NOP
;
663 /* Follow hash chain until the end. */
664 emit_move(as
, dest
, tmp1
);
666 emit_tsi(as
, MIPSI_LW
, tmp1
, dest
, (int32_t)offsetof(Node
, next
));
667 l_next
= emit_label(as
);
669 /* Type and value comparison. */
671 emit_branch(as
, MIPSI_BC1T
, 0, 0, l_end
);
672 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmpnum
, key
);
673 emit_tg(as
, MIPSI_MFC1
, tmp1
, key
+1);
674 emit_branch(as
, MIPSI_BEQ
, tmp1
, RID_ZERO
, l_next
);
675 emit_tsi(as
, MIPSI_SLTIU
, tmp1
, tmp1
, (int32_t)LJ_TISNUM
);
676 emit_hsi(as
, MIPSI_LDC1
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
679 emit_branch(as
, MIPSI_BEQ
, tmp1
, type
, l_end
);
681 emit_branch(as
, MIPSI_BEQ
, tmp2
, key
, l_end
);
682 emit_tsi(as
, MIPSI_LW
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
683 emit_branch(as
, MIPSI_BNE
, tmp1
, type
, l_next
);
686 emit_tsi(as
, MIPSI_LW
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
687 *l_loop
= MIPSI_BNE
| MIPSF_S(tmp1
) | ((as
->mcp
-l_loop
-1) & 0xffffu
);
689 /* Load main position relative to tab->node into dest. */
690 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
692 emit_tsi(as
, MIPSI_LW
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
695 if (irref_isk(refkey
))
696 tmphash
= ra_allock(as
, khash
, allow
);
697 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, tmp1
);
698 lua_assert(sizeof(Node
) == 24);
699 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp2
, tmp1
);
700 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, 3);
701 emit_dta(as
, MIPSI_SLL
, tmp2
, tmp1
, 5);
702 emit_dst(as
, MIPSI_AND
, tmp1
, tmp2
, tmphash
);
703 emit_tsi(as
, MIPSI_LW
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
704 emit_tsi(as
, MIPSI_LW
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
705 if (irref_isk(refkey
)) {
707 } else if (irt_isstr(kt
)) {
708 emit_tsi(as
, MIPSI_LW
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
709 } else { /* Must match with hash*() in lj_tab.c. */
710 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp1
, tmp2
);
711 emit_rotr(as
, tmp2
, tmp2
, dest
, (-HASH_ROT3
)&31);
712 emit_dst(as
, MIPSI_XOR
, tmp1
, tmp1
, tmp2
);
713 emit_rotr(as
, tmp1
, tmp1
, dest
, (-HASH_ROT2
-HASH_ROT1
)&31);
714 emit_dst(as
, MIPSI_SUBU
, tmp2
, tmp2
, dest
);
716 emit_dst(as
, MIPSI_XOR
, tmp2
, tmp2
, tmp1
);
717 if ((as
->flags
& JIT_F_MIPS32R2
)) {
718 emit_dta(as
, MIPSI_ROTR
, dest
, tmp1
, (-HASH_ROT1
)&31);
720 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp1
);
721 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, HASH_ROT1
);
722 emit_dta(as
, MIPSI_SRL
, dest
, tmp1
, (-HASH_ROT1
)&31);
724 emit_dst(as
, MIPSI_ADDU
, tmp1
, tmp1
, tmp1
);
725 emit_tg(as
, MIPSI_MFC1
, tmp2
, key
);
726 emit_tg(as
, MIPSI_MFC1
, tmp1
, key
+1);
728 emit_dst(as
, MIPSI_XOR
, tmp2
, key
, tmp1
);
729 emit_rotr(as
, dest
, tmp1
, tmp2
, (-HASH_ROT1
)&31);
730 emit_dst(as
, MIPSI_ADDU
, tmp1
, key
, ra_allock(as
, HASH_BIAS
, allow
));
736 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
738 IRIns
*kslot
= IR(ir
->op2
);
739 IRIns
*irkey
= IR(kslot
->op1
);
740 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
741 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
742 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
743 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
744 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
745 RegSet allow
= rset_exclude(RSET_GPR
, node
);
747 lua_assert(ofs
% sizeof(Node
) == 0);
750 rset_clear(allow
, dest
);
751 kofs
= (int32_t)offsetof(Node
, key
);
752 } else if (ra_hasreg(dest
)) {
753 emit_tsi(as
, MIPSI_ADDIU
, dest
, node
, ofs
);
755 if (!irt_ispri(irkey
->t
)) {
756 key
= ra_scratch(as
, allow
);
757 rset_clear(allow
, key
);
759 if (irt_isnum(irkey
->t
)) {
760 lo
= (int32_t)ir_knum(irkey
)->u32
.lo
;
761 hi
= (int32_t)ir_knum(irkey
)->u32
.hi
;
764 hi
= irt_toitype(irkey
->t
);
768 asm_guard(as
, MIPSI_BNE
, key
, lo
? ra_allock(as
, lo
, allow
) : RID_ZERO
);
770 asm_guard(as
, MIPSI_BNE
, type
, hi
? ra_allock(as
, hi
, allow
) : RID_ZERO
);
771 if (ra_hasreg(key
)) emit_tsi(as
, MIPSI_LW
, key
, idx
, kofs
+(LJ_BE
?4:0));
772 emit_tsi(as
, MIPSI_LW
, type
, idx
, kofs
+(LJ_BE
?0:4));
774 emit_tsi(as
, MIPSI_ADDU
, dest
, node
, ra_allock(as
, ofs
, allow
));
777 static void asm_newref(ASMState
*as
, IRIns
*ir
)
779 if (ir
->r
!= RID_SINK
) {
780 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
782 args
[0] = ASMREF_L
; /* lua_State *L */
783 args
[1] = ir
->op1
; /* GCtab *t */
784 args
[2] = ASMREF_TMP1
; /* cTValue *key */
785 asm_setupresult(as
, ir
, ci
); /* TValue * */
786 asm_gencall(as
, ci
, args
);
787 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op2
);
791 static void asm_uref(ASMState
*as
, IRIns
*ir
)
793 /* NYI: Check that UREFO is still open and not aliasing a slot. */
794 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
795 if (irref_isk(ir
->op1
)) {
796 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
797 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
798 emit_lsptr(as
, MIPSI_LW
, dest
, v
, RSET_GPR
);
800 Reg uv
= ra_scratch(as
, RSET_GPR
);
801 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
802 if (ir
->o
== IR_UREFC
) {
803 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
804 emit_tsi(as
, MIPSI_ADDIU
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
805 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
807 emit_tsi(as
, MIPSI_LW
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
809 emit_tsi(as
, MIPSI_LW
, uv
, func
,
810 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
814 static void asm_fref(ASMState
*as
, IRIns
*ir
)
816 UNUSED(as
); UNUSED(ir
);
817 lua_assert(!ra_used(ir
));
820 static void asm_strref(ASMState
*as
, IRIns
*ir
)
822 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
823 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
824 int32_t ofs
= (int32_t)sizeof(GCstr
);
826 if (irref_isk(ref
)) {
827 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
828 } else if (!irref_isk(refk
)) {
829 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
830 IRIns
*irr
= IR(ir
->op2
);
831 if (ra_hasreg(irr
->r
)) {
832 ra_noweak(as
, irr
->r
);
834 } else if (mayfuse(as
, irr
->op2
) &&
835 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
836 checki16(ofs
+ IR(irr
->op2
)->i
)) {
837 ofs
+= IR(irr
->op2
)->i
;
838 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
840 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
842 emit_tsi(as
, MIPSI_ADDIU
, dest
, dest
, ofs
);
843 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
846 r
= ra_alloc1(as
, ref
, RSET_GPR
);
849 emit_tsi(as
, MIPSI_ADDIU
, dest
, r
, ofs
);
851 emit_dst(as
, MIPSI_ADDU
, dest
, r
,
852 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
855 /* -- Loads and stores ---------------------------------------------------- */
857 static MIPSIns
asm_fxloadins(IRIns
*ir
)
859 switch (irt_type(ir
->t
)) {
860 case IRT_I8
: return MIPSI_LB
;
861 case IRT_U8
: return MIPSI_LBU
;
862 case IRT_I16
: return MIPSI_LH
;
863 case IRT_U16
: return MIPSI_LHU
;
864 case IRT_NUM
: return MIPSI_LDC1
;
865 case IRT_FLOAT
: return MIPSI_LWC1
;
866 default: return MIPSI_LW
;
870 static MIPSIns
asm_fxstoreins(IRIns
*ir
)
872 switch (irt_type(ir
->t
)) {
873 case IRT_I8
: case IRT_U8
: return MIPSI_SB
;
874 case IRT_I16
: case IRT_U16
: return MIPSI_SH
;
875 case IRT_NUM
: return MIPSI_SDC1
;
876 case IRT_FLOAT
: return MIPSI_SWC1
;
877 default: return MIPSI_SW
;
881 static void asm_fload(ASMState
*as
, IRIns
*ir
)
883 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
884 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
885 MIPSIns mi
= asm_fxloadins(ir
);
887 if (ir
->op2
== IRFL_TAB_ARRAY
) {
888 ofs
= asm_fuseabase(as
, ir
->op1
);
889 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
890 emit_tsi(as
, MIPSI_ADDIU
, dest
, idx
, ofs
);
894 ofs
= field_ofs
[ir
->op2
];
895 lua_assert(!irt_isfp(ir
->t
));
896 emit_tsi(as
, mi
, dest
, idx
, ofs
);
899 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
901 if (ir
->r
!= RID_SINK
) {
902 Reg src
= ra_alloc1z(as
, ir
->op2
, RSET_GPR
);
903 IRIns
*irf
= IR(ir
->op1
);
904 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
905 int32_t ofs
= field_ofs
[irf
->op2
];
906 MIPSIns mi
= asm_fxstoreins(ir
);
907 lua_assert(!irt_isfp(ir
->t
));
908 emit_tsi(as
, mi
, src
, idx
, ofs
);
912 static void asm_xload(ASMState
*as
, IRIns
*ir
)
914 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
915 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
916 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
919 static void asm_xstore(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
921 if (ir
->r
!= RID_SINK
) {
922 Reg src
= ra_alloc1z(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
923 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
924 rset_exclude(RSET_GPR
, src
), ofs
);
928 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
931 Reg dest
= RID_NONE
, type
= RID_TMP
, idx
;
932 RegSet allow
= RSET_GPR
;
935 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
936 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
937 rset_clear(allow
, dest
);
939 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
940 rset_clear(allow
, idx
);
942 asm_guard(as
, MIPSI_BEQ
, type
, RID_ZERO
);
943 emit_tsi(as
, MIPSI_SLTIU
, type
, type
, (int32_t)LJ_TISNUM
);
945 emit_hsi(as
, MIPSI_LDC1
, dest
, idx
, ofs
);
947 asm_guard(as
, MIPSI_BNE
, type
, ra_allock(as
, irt_toitype(t
), allow
));
948 if (ra_hasreg(dest
)) emit_tsi(as
, MIPSI_LW
, dest
, idx
, ofs
+(LJ_BE
?4:0));
950 emit_tsi(as
, MIPSI_LW
, type
, idx
, ofs
+(LJ_BE
?0:4));
953 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
955 RegSet allow
= RSET_GPR
;
956 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
958 if (ir
->r
== RID_SINK
)
960 if (irt_isnum(ir
->t
)) {
961 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
963 if (!irt_ispri(ir
->t
)) {
964 src
= ra_alloc1(as
, ir
->op2
, allow
);
965 rset_clear(allow
, src
);
967 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
968 rset_clear(allow
, type
);
970 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
971 if (irt_isnum(ir
->t
)) {
972 emit_hsi(as
, MIPSI_SDC1
, src
, idx
, ofs
);
975 emit_tsi(as
, MIPSI_SW
, src
, idx
, ofs
+(LJ_BE
?4:0));
976 emit_tsi(as
, MIPSI_SW
, type
, idx
, ofs
+(LJ_BE
?0:4));
980 static void asm_sload(ASMState
*as
, IRIns
*ir
)
982 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 4 : 0);
984 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
985 RegSet allow
= RSET_GPR
;
986 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
987 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
988 lua_assert(!irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
989 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
990 dest
= ra_scratch(as
, RSET_FPR
);
991 asm_tointg(as
, ir
, dest
);
992 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
993 } else if (ra_used(ir
)) {
994 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
995 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
996 rset_clear(allow
, dest
);
997 base
= ra_alloc1(as
, REF_BASE
, allow
);
998 rset_clear(allow
, base
);
999 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1001 Reg tmp
= ra_scratch(as
, RSET_FPR
);
1002 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
1003 emit_fg(as
, MIPSI_CVT_W_D
, tmp
, tmp
);
1005 t
.irt
= IRT_NUM
; /* Check for original type. */
1007 Reg tmp
= ra_scratch(as
, RSET_GPR
);
1008 emit_fg(as
, MIPSI_CVT_D_W
, dest
, dest
);
1009 emit_tg(as
, MIPSI_MTC1
, tmp
, dest
);
1011 t
.irt
= IRT_INT
; /* Check for original type. */
1016 base
= ra_alloc1(as
, REF_BASE
, allow
);
1017 rset_clear(allow
, base
);
1020 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1021 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1022 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)LJ_TISNUM
);
1025 if (ra_hasreg(dest
)) emit_hsi(as
, MIPSI_LDC1
, dest
, base
, ofs
);
1027 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1028 Reg ktype
= ra_allock(as
, irt_toitype(t
), allow
);
1029 asm_guard(as
, MIPSI_BNE
, RID_TMP
, ktype
);
1032 if (ra_hasreg(dest
)) emit_tsi(as
, MIPSI_LW
, dest
, base
, ofs
^ (LJ_BE
?4:0));
1034 if (ra_hasreg(type
)) emit_tsi(as
, MIPSI_LW
, type
, base
, ofs
^ (LJ_BE
?0:4));
1037 /* -- Allocations --------------------------------------------------------- */
1040 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1042 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1043 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1044 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1045 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1046 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1048 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1049 RegSet drop
= RSET_SCRATCH
;
1050 lua_assert(sz
!= CTSIZE_INVALID
);
1052 args
[0] = ASMREF_L
; /* lua_State *L */
1053 args
[1] = ASMREF_TMP1
; /* MSize size */
1056 if (ra_hasreg(ir
->r
))
1057 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1058 ra_evictset(as
, drop
);
1060 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1062 /* Initialize immutable cdata object. */
1063 if (ir
->o
== IR_CNEWI
) {
1064 int32_t ofs
= sizeof(GCcdata
);
1065 lua_assert(sz
== 4 || sz
== 8);
1068 lua_assert((ir
+1)->o
== IR_HIOP
);
1072 Reg r
= ra_alloc1z(as
, ir
->op2
, allow
);
1073 emit_tsi(as
, MIPSI_SW
, r
, RID_RET
, ofs
);
1074 rset_clear(allow
, r
);
1075 if (ofs
== sizeof(GCcdata
)) break;
1076 ofs
-= 4; if (LJ_BE
) ir
++; else ir
--;
1079 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1080 emit_tsi(as
, MIPSI_SB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1081 emit_tsi(as
, MIPSI_SH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1082 emit_ti(as
, MIPSI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1083 emit_ti(as
, MIPSI_LI
, RID_TMP
, ctypeid
); /* Lower 16 bit used. Sign-ext ok. */
1084 asm_gencall(as
, ci
, args
);
1085 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1086 ra_releasetmp(as
, ASMREF_TMP1
));
1089 #define asm_cnew(as, ir) ((void)0)
1092 /* -- Write barriers ------------------------------------------------------ */
1094 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1096 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1097 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1099 MCLabel l_end
= emit_label(as
);
1100 emit_tsi(as
, MIPSI_SW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1101 emit_tsi(as
, MIPSI_SB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1102 emit_setgl(as
, tab
, gc
.grayagain
);
1103 emit_getgl(as
, link
, gc
.grayagain
);
1104 emit_dst(as
, MIPSI_XOR
, mark
, mark
, RID_TMP
); /* Clear black bit. */
1105 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1106 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, mark
, LJ_GC_BLACK
);
1107 emit_tsi(as
, MIPSI_LBU
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1110 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1112 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1116 /* No need for other object barriers (yet). */
1117 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1118 ra_evictset(as
, RSET_SCRATCH
);
1119 l_end
= emit_label(as
);
1120 args
[0] = ASMREF_TMP1
; /* global_State *g */
1121 args
[1] = ir
->op1
; /* TValue *tv */
1122 asm_gencall(as
, ci
, args
);
1123 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1124 obj
= IR(ir
->op1
)->r
;
1125 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1126 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1127 emit_tsi(as
, MIPSI_ANDI
, tmp
, tmp
, LJ_GC_BLACK
);
1128 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1129 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1130 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1131 emit_tsi(as
, MIPSI_LBU
, tmp
, obj
,
1132 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1133 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1136 /* -- Arithmetic and logic operations ------------------------------------- */
1138 static void asm_fparith(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1140 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1141 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1142 right
= (left
>> 8); left
&= 255;
1143 emit_fgh(as
, mi
, dest
, left
, right
);
1146 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1148 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1149 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1150 emit_fg(as
, mi
, dest
, left
);
1153 static int asm_fpjoin_pow(ASMState
*as
, IRIns
*ir
)
1155 IRIns
*irp
= IR(ir
->op1
);
1156 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1157 IRIns
*irpp
= IR(irp
->op1
);
1158 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1159 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1160 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_pow
];
1162 args
[0] = irpp
->op1
;
1164 asm_setupresult(as
, ir
, ci
);
1165 asm_gencall(as
, ci
, args
);
1172 static void asm_add(ASMState
*as
, IRIns
*ir
)
1174 if (irt_isnum(ir
->t
)) {
1175 asm_fparith(as
, ir
, MIPSI_ADD_D
);
1177 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1178 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1179 if (irref_isk(ir
->op2
)) {
1180 int32_t k
= IR(ir
->op2
)->i
;
1182 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1186 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1187 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1191 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1193 if (irt_isnum(ir
->t
)) {
1194 asm_fparith(as
, ir
, MIPSI_SUB_D
);
1196 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1197 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1198 right
= (left
>> 8); left
&= 255;
1199 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1203 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1205 if (irt_isnum(ir
->t
)) {
1206 asm_fparith(as
, ir
, MIPSI_MUL_D
);
1208 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1209 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1210 right
= (left
>> 8); left
&= 255;
1211 emit_dst(as
, MIPSI_MUL
, dest
, left
, right
);
1215 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1217 if (irt_isnum(ir
->t
)) {
1218 asm_fpunary(as
, ir
, MIPSI_NEG_D
);
1220 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1221 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1222 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1226 static void asm_arithov(ASMState
*as
, IRIns
*ir
)
1228 Reg right
, left
, tmp
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1229 if (irref_isk(ir
->op2
)) {
1230 int k
= IR(ir
->op2
)->i
;
1231 if (ir
->o
== IR_SUBOV
) k
= -k
;
1232 if (checki16(k
)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1233 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1234 asm_guard(as
, k
>= 0 ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1235 emit_dst(as
, MIPSI_SLT
, RID_TMP
, dest
, dest
== left
? RID_TMP
: left
);
1236 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1237 if (dest
== left
) emit_move(as
, RID_TMP
, left
);
1241 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1242 right
= (left
>> 8); left
&= 255;
1243 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
, left
),
1245 asm_guard(as
, MIPSI_BLTZ
, RID_TMP
, 0);
1246 emit_dst(as
, MIPSI_AND
, RID_TMP
, RID_TMP
, tmp
);
1247 if (ir
->o
== IR_ADDOV
) { /* ((dest^left) & (dest^right)) < 0 */
1248 emit_dst(as
, MIPSI_XOR
, RID_TMP
, dest
, dest
== right
? RID_TMP
: right
);
1249 } else { /* ((dest^left) & (dest^~right)) < 0 */
1250 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, dest
);
1251 emit_dst(as
, MIPSI_NOR
, RID_TMP
, dest
== right
? RID_TMP
: right
, RID_ZERO
);
1253 emit_dst(as
, MIPSI_XOR
, tmp
, dest
, dest
== left
? RID_TMP
: left
);
1254 emit_dst(as
, ir
->o
== IR_ADDOV
? MIPSI_ADDU
: MIPSI_SUBU
, dest
, left
, right
);
1255 if (dest
== left
|| dest
== right
)
1256 emit_move(as
, RID_TMP
, dest
== left
? left
: right
);
1259 static void asm_mulov(ASMState
*as
, IRIns
*ir
)
1264 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused in single-number mode. */
1269 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1271 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1272 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1273 if (irref_isk(ir
->op2
)) {
1274 int32_t k
= IR(ir
->op2
)->i
;
1276 emit_dst(as
, MIPSI_ADDU
, dest
, left
, RID_TMP
);
1278 } else if (checki16(k
)) {
1279 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1280 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1284 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1285 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1286 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1289 dest
= ra_dest(as
, ir
, RSET_GPR
);
1290 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1291 if (irref_isk(ir
->op2
)) {
1292 int32_t k
= IR(ir
->op2
)->i
;
1295 emit_move(as
, dest
, left
);
1297 } else if (checki16(k
)) {
1299 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, left
));
1300 emit_move(as
, dest
, tmp
);
1303 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, left
);
1304 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1308 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1309 if (dest
== left
&& dest
== right
) {
1310 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1311 emit_move(as
, dest
, tmp
);
1314 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, dest
== left
? right
: left
);
1315 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1318 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1320 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1321 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1322 right
= (left
>> 8); left
&= 255;
1323 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
1324 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1326 dest
= ra_dest(as
, ir
, RSET_GPR
);
1327 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1328 right
= (left
>> 8); left
&= 255;
1330 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1331 emit_move(as
, dest
, tmp
);
1334 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, left
, dest
);
1335 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1338 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1340 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1341 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1342 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
1343 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1345 dest
= ra_dest(as
, ir
, RSET_GPR
);
1346 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1347 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_ZERO
, dest
);
1348 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1352 static void asm_bitnot(ASMState
*as
, IRIns
*ir
)
1354 Reg left
, right
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1355 IRIns
*irl
= IR(ir
->op1
);
1356 if (mayfuse(as
, ir
->op1
) && irl
->o
== IR_BOR
) {
1357 left
= ra_alloc2(as
, irl
, RSET_GPR
);
1358 right
= (left
>> 8); left
&= 255;
1360 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1363 emit_dst(as
, MIPSI_NOR
, dest
, left
, right
);
1366 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1368 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1369 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1370 if ((as
->flags
& JIT_F_MIPS32R2
)) {
1371 emit_dta(as
, MIPSI_ROTR
, dest
, RID_TMP
, 16);
1372 emit_dst(as
, MIPSI_WSBH
, RID_TMP
, 0, left
);
1374 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), dest
));
1375 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp
);
1376 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
1377 emit_tsi(as
, MIPSI_ANDI
, dest
, dest
, 0xff00);
1378 emit_dta(as
, MIPSI_SLL
, RID_TMP
, RID_TMP
, 8);
1379 emit_dta(as
, MIPSI_SRL
, dest
, left
, 8);
1380 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, left
, 0xff00);
1381 emit_dst(as
, MIPSI_OR
, tmp
, tmp
, RID_TMP
);
1382 emit_dta(as
, MIPSI_SRL
, tmp
, left
, 24);
1383 emit_dta(as
, MIPSI_SLL
, RID_TMP
, left
, 24);
1387 static void asm_bitop(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
1389 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1390 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1391 if (irref_isk(ir
->op2
)) {
1392 int32_t k
= IR(ir
->op2
)->i
;
1394 emit_tsi(as
, mik
, dest
, left
, k
);
1398 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1399 emit_dst(as
, mi
, dest
, left
, right
);
1402 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
1404 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1405 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1406 uint32_t shift
= (uint32_t)(IR(ir
->op2
)->i
& 31);
1407 emit_dta(as
, mik
, dest
, ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
), shift
);
1409 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1410 right
= (left
>> 8); left
&= 255;
1411 emit_dst(as
, mi
, dest
, right
, left
); /* Shift amount is in rs. */
1415 static void asm_bitror(ASMState
*as
, IRIns
*ir
)
1417 if ((as
->flags
& JIT_F_MIPS32R2
)) {
1418 asm_bitshift(as
, ir
, MIPSI_ROTRV
, MIPSI_ROTR
);
1420 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1421 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1422 uint32_t shift
= (uint32_t)(IR(ir
->op2
)->i
& 31);
1423 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1424 emit_rotr(as
, dest
, left
, RID_TMP
, shift
);
1426 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1427 right
= (left
>> 8); left
&= 255;
1428 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
1429 emit_dst(as
, MIPSI_SRLV
, dest
, right
, left
);
1430 emit_dst(as
, MIPSI_SLLV
, RID_TMP
, RID_TMP
, left
);
1431 emit_dst(as
, MIPSI_SUBU
, RID_TMP
, ra_allock(as
, 32, RSET_GPR
), right
);
1436 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1438 if (irt_isnum(ir
->t
)) {
1439 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1440 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1441 right
= (left
>> 8); left
&= 255;
1443 emit_fg(as
, MIPSI_MOVT_D
, dest
, right
);
1445 emit_fg(as
, MIPSI_MOVF_D
, dest
, left
);
1446 if (dest
!= right
) emit_fg(as
, MIPSI_MOV_D
, dest
, right
);
1448 emit_fgh(as
, MIPSI_C_OLT_D
, 0, ismax
? left
: right
, ismax
? right
: left
);
1450 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1451 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1452 right
= (left
>> 8); left
&= 255;
1454 emit_dst(as
, MIPSI_MOVN
, dest
, right
, RID_TMP
);
1456 emit_dst(as
, MIPSI_MOVZ
, dest
, left
, RID_TMP
);
1457 if (dest
!= right
) emit_move(as
, dest
, right
);
1459 emit_dst(as
, MIPSI_SLT
, RID_TMP
,
1460 ismax
? left
: right
, ismax
? right
: left
);
1464 /* -- Comparisons --------------------------------------------------------- */
1466 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1468 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1470 if (irt_isnum(ir
->t
)) {
1471 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1472 right
= (left
>> 8); left
&= 255;
1473 asm_guard(as
, (op
&1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
1474 emit_fgh(as
, MIPSI_C_OLT_D
+ ((op
&3) ^ ((op
>>2)&1)), 0, left
, right
);
1476 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1477 if (op
== IR_ABC
) op
= IR_UGT
;
1478 if ((op
&4) == 0 && irref_isk(ir
->op2
) && IR(ir
->op2
)->i
== 0) {
1479 MIPSIns mi
= (op
&2) ? ((op
&1) ? MIPSI_BLEZ
: MIPSI_BGTZ
) :
1480 ((op
&1) ? MIPSI_BLTZ
: MIPSI_BGEZ
);
1481 asm_guard(as
, mi
, left
, 0);
1483 if (irref_isk(ir
->op2
)) {
1484 int32_t k
= IR(ir
->op2
)->i
;
1487 asm_guard(as
, (op
&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1488 emit_tsi(as
, (op
&4) ? MIPSI_SLTIU
: MIPSI_SLTI
,
1493 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1494 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1495 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
,
1496 RID_TMP
, (op
&2) ? right
: left
, (op
&2) ? left
: right
);
1501 static void asm_compeq(ASMState
*as
, IRIns
*ir
)
1503 Reg right
, left
= ra_alloc2(as
, ir
, irt_isnum(ir
->t
) ? RSET_FPR
: RSET_GPR
);
1504 right
= (left
>> 8); left
&= 255;
1505 if (irt_isnum(ir
->t
)) {
1506 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
1507 emit_fgh(as
, MIPSI_C_EQ_D
, 0, left
, right
);
1509 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, left
, right
);
1514 /* 64 bit integer comparisons. */
1515 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1517 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1518 IROp op
= (ir
-1)->o
;
1520 Reg rightlo
, leftlo
, righthi
, lefthi
= ra_alloc2(as
, ir
, RSET_GPR
);
1521 righthi
= (lefthi
>> 8); lefthi
&= 255;
1522 leftlo
= ra_alloc2(as
, ir
-1,
1523 rset_exclude(rset_exclude(RSET_GPR
, lefthi
), righthi
));
1524 rightlo
= (leftlo
>> 8); leftlo
&= 255;
1525 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1526 l_end
= emit_label(as
);
1527 if (lefthi
!= righthi
)
1528 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
, RID_TMP
,
1529 (op
&2) ? righthi
: lefthi
, (op
&2) ? lefthi
: righthi
);
1530 emit_dst(as
, MIPSI_SLTU
, RID_TMP
,
1531 (op
&2) ? rightlo
: leftlo
, (op
&2) ? leftlo
: rightlo
);
1532 if (lefthi
!= righthi
)
1533 emit_branch(as
, MIPSI_BEQ
, lefthi
, righthi
, l_end
);
1536 static void asm_comp64eq(ASMState
*as
, IRIns
*ir
)
1538 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1539 right
= (left
>> 8); left
&= 255;
1540 asm_guard(as
, ((ir
-1)->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, RID_TMP
, RID_ZERO
);
1541 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1542 emit_dst(as
, MIPSI_OR
, RID_TMP
, RID_TMP
, tmp
);
1543 emit_dst(as
, MIPSI_XOR
, tmp
, left
, right
);
1544 left
= ra_alloc2(as
, ir
-1, RSET_GPR
);
1545 right
= (left
>> 8); left
&= 255;
1546 emit_dst(as
, MIPSI_XOR
, RID_TMP
, left
, right
);
1550 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1552 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1553 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1556 /* HIOP is marked as a store because it needs its own DCE logic. */
1557 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1558 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1559 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1560 as
->curins
--; /* Always skip the CONV. */
1564 } else if ((ir
-1)->o
< IR_EQ
) { /* 64 bit integer comparisons. ORDER IR. */
1565 as
->curins
--; /* Always skip the loword comparison. */
1568 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1569 as
->curins
--; /* Always skip the loword comparison. */
1570 asm_comp64eq(as
, ir
);
1572 } else if ((ir
-1)->o
== IR_XSTORE
) {
1573 as
->curins
--; /* Handle both stores here. */
1574 if ((ir
-1)->r
!= RID_SINK
) {
1575 asm_xstore(as
, ir
, LJ_LE
? 4 : 0);
1576 asm_xstore(as
, ir
-1, LJ_LE
? 0 : 4);
1580 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1581 switch ((ir
-1)->o
) {
1582 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1583 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1584 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1588 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1591 /* Nothing to do here. Handled by lo op itself. */
1593 default: lua_assert(0); break;
1596 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1600 /* -- Stack handling ------------------------------------------------------ */
1602 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1603 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1604 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1606 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1607 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1608 ExitNo oldsnap
= as
->snapno
;
1609 rset_clear(allow
, pbase
);
1610 tmp
= allow
? rset_pickbot(allow
) :
1611 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1612 as
->snapno
= exitno
;
1613 asm_guard(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
);
1614 as
->snapno
= oldsnap
;
1615 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1616 emit_tsi(as
, MIPSI_LW
, tmp
, RID_SP
, 0);
1618 ra_modified(as
, tmp
);
1619 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)(8*topslot
));
1620 emit_dst(as
, MIPSI_SUBU
, RID_TMP
, tmp
, pbase
);
1621 emit_tsi(as
, MIPSI_LW
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1622 if (pbase
== RID_TMP
)
1623 emit_getgl(as
, RID_TMP
, jit_base
);
1624 emit_getgl(as
, tmp
, jit_L
);
1625 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1626 emit_tsi(as
, MIPSI_SW
, tmp
, RID_SP
, 0);
1629 /* Restore Lua stack from on-trace state. */
1630 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1632 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1633 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1634 MSize n
, nent
= snap
->nent
;
1635 /* Store the value of all modified slots to the Lua stack. */
1636 for (n
= 0; n
< nent
; n
++) {
1637 SnapEntry sn
= map
[n
];
1638 BCReg s
= snap_slot(sn
);
1639 int32_t ofs
= 8*((int32_t)s
-1);
1640 IRRef ref
= snap_ref(sn
);
1641 IRIns
*ir
= IR(ref
);
1642 if ((sn
& SNAP_NORESTORE
))
1644 if (irt_isnum(ir
->t
)) {
1645 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1646 emit_hsi(as
, MIPSI_SDC1
, src
, RID_BASE
, ofs
);
1649 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1650 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1651 if (!irt_ispri(ir
->t
)) {
1652 Reg src
= ra_alloc1(as
, ref
, allow
);
1653 rset_clear(allow
, src
);
1654 emit_tsi(as
, MIPSI_SW
, src
, RID_BASE
, ofs
+(LJ_BE
?4:0));
1656 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1657 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1658 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1660 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1662 emit_tsi(as
, MIPSI_SW
, type
, RID_BASE
, ofs
+(LJ_BE
?0:4));
1666 lua_assert(map
+ nent
== flinks
);
1669 /* -- GC handling --------------------------------------------------------- */
1671 /* Check GC threshold and do one or more GC steps. */
1672 static void asm_gc_check(ASMState
*as
)
1674 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1678 ra_evictset(as
, RSET_SCRATCH
);
1679 l_end
= emit_label(as
);
1680 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1681 /* Assumes asm_snap_prep() already done. */
1682 asm_guard(as
, MIPSI_BNE
, RID_RET
, RID_ZERO
);
1683 args
[0] = ASMREF_TMP1
; /* global_State *g */
1684 args
[1] = ASMREF_TMP2
; /* MSize steps */
1685 asm_gencall(as
, ci
, args
);
1686 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1687 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1688 emit_loadi(as
, tmp
, as
->gcsteps
);
1689 /* Jump around GC step if GC total < GC threshold. */
1690 emit_branch(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
, l_end
);
1691 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_TMP
, tmp
);
1692 emit_getgl(as
, tmp
, gc
.threshold
);
1693 emit_getgl(as
, RID_TMP
, gc
.total
);
1698 /* -- Loop handling ------------------------------------------------------- */
1700 /* Fixup the loop branch. */
1701 static void asm_loop_fixup(ASMState
*as
)
1703 MCode
*p
= as
->mctop
;
1704 MCode
*target
= as
->mcp
;
1706 if (as
->loopinv
) { /* Inverted loop branch? */
1707 /* asm_guard already inverted the cond branch. Only patch the target. */
1708 p
[-3] |= ((target
-p
+2) & 0x0000ffffu
);
1710 p
[-2] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1714 /* -- Head of trace ------------------------------------------------------- */
1716 /* Coalesce BASE register for a root trace. */
1717 static void asm_head_root_base(ASMState
*as
)
1719 IRIns
*ir
= IR(REF_BASE
);
1721 if (as
->loopinv
) as
->mctop
--;
1724 if (rset_test(as
->modset
, r
))
1725 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1727 emit_move(as
, r
, RID_BASE
);
1731 /* Coalesce BASE register for a side trace. */
1732 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1734 IRIns
*ir
= IR(REF_BASE
);
1736 if (as
->loopinv
) as
->mctop
--;
1739 if (rset_test(as
->modset
, r
))
1740 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1742 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1743 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1744 rset_clear(allow
, irp
->r
);
1745 emit_move(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1747 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1753 /* -- Tail of trace ------------------------------------------------------- */
1755 /* Fixup the tail code. */
1756 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1758 MCode
*target
= lnk
? traceref(as
->J
,lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1759 int32_t spadj
= as
->T
->spadjust
;
1760 MCode
*p
= as
->mctop
-1;
1761 *p
= spadj
? (MIPSI_ADDIU
|MIPSF_T(RID_SP
)|MIPSF_S(RID_SP
)|spadj
) : MIPSI_NOP
;
1762 p
[-1] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1765 /* Prepare tail of code. */
1766 static void asm_tail_prep(ASMState
*as
)
1768 as
->mcp
= as
->mctop
-2; /* Leave room for branch plus nop or stack adj. */
1769 as
->invmcp
= as
->loopref
? as
->mcp
: NULL
;
1772 /* -- Instruction dispatch ------------------------------------------------ */
1774 /* Assemble a single instruction. */
1775 static void asm_ir(ASMState
*as
, IRIns
*ir
)
1777 switch ((IROp
)ir
->o
) {
1778 /* Miscellaneous ops. */
1779 case IR_LOOP
: asm_loop(as
); break;
1780 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
1782 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
1783 case IR_PHI
: asm_phi(as
, ir
); break;
1784 case IR_HIOP
: asm_hiop(as
, ir
); break;
1785 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
1787 /* Guarded assertions. */
1788 case IR_EQ
: case IR_NE
: asm_compeq(as
, ir
); break;
1789 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
1790 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
1795 case IR_RETF
: asm_retf(as
, ir
); break;
1798 case IR_BNOT
: asm_bitnot(as
, ir
); break;
1799 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
1801 case IR_BAND
: asm_bitop(as
, ir
, MIPSI_AND
, MIPSI_ANDI
); break;
1802 case IR_BOR
: asm_bitop(as
, ir
, MIPSI_OR
, MIPSI_ORI
); break;
1803 case IR_BXOR
: asm_bitop(as
, ir
, MIPSI_XOR
, MIPSI_XORI
); break;
1805 case IR_BSHL
: asm_bitshift(as
, ir
, MIPSI_SLLV
, MIPSI_SLL
); break;
1806 case IR_BSHR
: asm_bitshift(as
, ir
, MIPSI_SRLV
, MIPSI_SRL
); break;
1807 case IR_BSAR
: asm_bitshift(as
, ir
, MIPSI_SRAV
, MIPSI_SRA
); break;
1808 case IR_BROL
: lua_assert(0); break;
1809 case IR_BROR
: asm_bitror(as
, ir
); break;
1811 /* Arithmetic ops. */
1812 case IR_ADD
: asm_add(as
, ir
); break;
1813 case IR_SUB
: asm_sub(as
, ir
); break;
1814 case IR_MUL
: asm_mul(as
, ir
); break;
1815 case IR_DIV
: asm_fparith(as
, ir
, MIPSI_DIV_D
); break;
1816 case IR_MOD
: asm_callid(as
, ir
, IRCALL_lj_vm_modi
); break;
1817 case IR_POW
: asm_callid(as
, ir
, IRCALL_lj_vm_powi
); break;
1818 case IR_NEG
: asm_neg(as
, ir
); break;
1820 case IR_ABS
: asm_fpunary(as
, ir
, MIPSI_ABS_D
); break;
1821 case IR_ATAN2
: asm_callid(as
, ir
, IRCALL_atan2
); break;
1822 case IR_LDEXP
: asm_callid(as
, ir
, IRCALL_ldexp
); break;
1823 case IR_MIN
: asm_min_max(as
, ir
, 0); break;
1824 case IR_MAX
: asm_min_max(as
, ir
, 1); break;
1826 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
1828 if (ir
->op2
<= IRFPM_TRUNC
)
1829 asm_callround(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1830 else if (ir
->op2
== IRFPM_SQRT
)
1831 asm_fpunary(as
, ir
, MIPSI_SQRT_D
);
1833 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1836 /* Overflow-checking arithmetic ops. */
1837 case IR_ADDOV
: asm_arithov(as
, ir
); break;
1838 case IR_SUBOV
: asm_arithov(as
, ir
); break;
1839 case IR_MULOV
: asm_mulov(as
, ir
); break;
1841 /* Memory references. */
1842 case IR_AREF
: asm_aref(as
, ir
); break;
1843 case IR_HREF
: asm_href(as
, ir
); break;
1844 case IR_HREFK
: asm_hrefk(as
, ir
); break;
1845 case IR_NEWREF
: asm_newref(as
, ir
); break;
1846 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
1847 case IR_FREF
: asm_fref(as
, ir
); break;
1848 case IR_STRREF
: asm_strref(as
, ir
); break;
1850 /* Loads and stores. */
1851 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
1852 asm_ahuvload(as
, ir
);
1854 case IR_FLOAD
: asm_fload(as
, ir
); break;
1855 case IR_XLOAD
: asm_xload(as
, ir
); break;
1856 case IR_SLOAD
: asm_sload(as
, ir
); break;
1858 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
1859 case IR_FSTORE
: asm_fstore(as
, ir
); break;
1860 case IR_XSTORE
: asm_xstore(as
, ir
, 0); break;
1863 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
1864 case IR_TNEW
: asm_tnew(as
, ir
); break;
1865 case IR_TDUP
: asm_tdup(as
, ir
); break;
1866 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
1868 /* Write barriers. */
1869 case IR_TBAR
: asm_tbar(as
, ir
); break;
1870 case IR_OBAR
: asm_obar(as
, ir
); break;
1872 /* Type conversions. */
1873 case IR_CONV
: asm_conv(as
, ir
); break;
1874 case IR_TOBIT
: asm_tobit(as
, ir
); break;
1875 case IR_TOSTR
: asm_tostr(as
, ir
); break;
1876 case IR_STRTO
: asm_strto(as
, ir
); break;
1879 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
1880 case IR_CALLXS
: asm_callx(as
, ir
); break;
1881 case IR_CARG
: break;
1884 setintV(&as
->J
->errinfo
, ir
->o
);
1885 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
1890 /* -- Trace setup --------------------------------------------------------- */
1892 /* Ensure there are enough stack slots for call arguments. */
1893 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
1895 IRRef args
[CCI_NARGS_MAX
];
1896 uint32_t i
, nargs
= (int)CCI_NARGS(ci
);
1897 int nslots
= 4, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
1898 asm_collectargs(as
, ir
, ci
, args
);
1899 for (i
= 0; i
< nargs
; i
++) {
1900 if (args
[i
] && irt_isfp(IR(args
[i
])->t
) &&
1901 nfpr
> 0 && !(ci
->flags
& CCI_VARARG
)) {
1903 ngpr
-= irt_isnum(IR(args
[i
])->t
) ? 2 : 1;
1904 } else if (args
[i
] && irt_isnum(IR(args
[i
])->t
)) {
1907 if (ngpr
> 0) ngpr
-= 2; else nslots
= (nslots
+3) & ~1;
1910 if (ngpr
> 0) ngpr
--; else nslots
++;
1913 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
1914 as
->evenspill
= nslots
;
1915 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
1918 static void asm_setup_target(ASMState
*as
)
1920 asm_sparejump_setup(as
);
1921 asm_exitstub_setup(as
);
1924 /* -- Trace patching ------------------------------------------------------ */
1926 /* Patch exit jumps of existing machine code to a new target. */
1927 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
1929 MCode
*p
= T
->mcode
;
1930 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
1931 MCode
*px
= exitstub_trace_addr(T
, exitno
);
1932 MCode
*cstart
= NULL
, *cstop
= NULL
;
1933 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
1934 MCode exitload
= MIPSI_LI
| MIPSF_T(RID_TMP
) | exitno
;
1935 MCode tjump
= MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1936 for (p
++; p
< pe
; p
++) {
1937 if (*p
== exitload
) { /* Look for load of exit number. */
1938 if (((p
[-1] ^ (px
-p
)) & 0xffffu
) == 0) { /* Look for exitstub branch. */
1939 ptrdiff_t delta
= target
- p
;
1940 if (((delta
+ 0x8000) >> 16) == 0) { /* Patch in-range branch. */
1942 p
[-1] = (p
[-1] & 0xffff0000u
) | (delta
& 0xffffu
);
1943 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
1945 if (!cstart
) cstart
= p
-1;
1946 } else { /* Branch out of range. Use spare jump slot in mcarea. */
1948 for (i
= 2; i
< 2+MIPS_SPAREJUMP
*2; i
+= 2) {
1949 if (mcarea
[i
] == tjump
) {
1950 delta
= mcarea
+i
- p
;
1952 } else if (mcarea
[i
] == MIPSI_NOP
) {
1955 delta
= mcarea
+i
- p
;
1959 /* Ignore jump slot overflow. Child trace is simply not attached. */
1961 } else if (p
+1 == pe
) {
1962 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
1963 lua_assert(p
[1] == MIPSI_NOP
);
1965 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
1967 if (!cstart
) cstart
= p
+1;
1971 if (cstart
) lj_mcode_sync(cstart
, cstop
);
1972 lj_mcode_patch(J
, mcarea
, 1);