2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate a register or RID_ZERO. */
22 static Reg
ra_alloc1z(ASMState
*as
, IRRef ref
, RegSet allow
)
26 if (!(allow
& RSET_FPR
) && irref_isk(ref
) && IR(ref
)->i
== 0)
28 r
= ra_allocref(as
, ref
, allow
);
35 /* Allocate two source registers for three-operand instructions. */
36 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
38 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
39 Reg left
= irl
->r
, right
= irr
->r
;
40 if (ra_hasreg(left
)) {
43 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
46 } else if (ra_hasreg(right
)) {
48 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
49 } else if (ra_hashint(right
)) {
50 right
= ra_alloc1z(as
, ir
->op2
, allow
);
51 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
53 left
= ra_alloc1z(as
, ir
->op1
, allow
);
54 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
56 return left
| (right
<< 8);
59 /* -- Guard handling ------------------------------------------------------ */
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
64 /* Setup spare long-range jump slots per mcarea. */
65 static void asm_sparejump_setup(ASMState
*as
)
67 MCode
*mxp
= as
->mcbot
;
68 /* Assumes sizeof(MCLink) == 8. */
69 if (((uintptr_t)mxp
& (LJ_PAGESIZE
-1)) == 8) {
70 lua_assert(MIPSI_NOP
== 0);
71 memset(mxp
+2, 0, MIPS_SPAREJUMP
*8);
72 mxp
+= MIPS_SPAREJUMP
*2;
73 lua_assert(mxp
< as
->mctop
);
74 lj_mcode_sync(as
->mcbot
, mxp
);
75 lj_mcode_commitbot(as
->J
, mxp
);
77 as
->mclim
= as
->mcbot
+ MCLIM_REDZONE
;
81 /* Setup exit stub after the end of each trace. */
82 static void asm_exitstub_setup(ASMState
*as
)
84 MCode
*mxp
= as
->mctop
;
85 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
86 *--mxp
= MIPSI_LI
|MIPSF_T(RID_TMP
)|as
->T
->traceno
;
87 *--mxp
= MIPSI_J
|((((uintptr_t)(void *)lj_vm_exit_handler
)>>2)&0x03ffffffu
);
88 lua_assert(((uintptr_t)mxp
^ (uintptr_t)(void *)lj_vm_exit_handler
)>>28 == 0);
89 *--mxp
= MIPSI_SW
|MIPSF_T(RID_TMP
)|MIPSF_S(RID_SP
)|0;
93 /* Keep this in-sync with exitstub_trace_addr(). */
94 #define asm_exitstub_addr(as) ((as)->mctop)
96 /* Emit conditional branch to exit for guard. */
97 static void asm_guard(ASMState
*as
, MIPSIns mi
, Reg rs
, Reg rt
)
99 MCode
*target
= asm_exitstub_addr(as
);
101 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
105 mi
= mi
^ ((mi
>>28) == 1 ? 0x04000000u
: 0x00010000u
); /* Invert cond. */
106 target
= p
; /* Patch target later in asm_loop_fixup. */
108 emit_ti(as
, MIPSI_LI
, RID_TMP
, as
->snapno
);
109 emit_branch(as
, mi
, rs
, rt
, target
);
112 /* -- Operand fusion ------------------------------------------------------ */
114 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
115 #define CONFLICT_SEARCH_LIM 31
117 /* Check if there's no conflicting instruction between curins and ref. */
118 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
121 IRRef i
= as
->curins
;
122 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
123 return 0; /* Give up, ref is too far away. */
125 if (ir
[i
].o
== conflict
)
126 return 0; /* Conflict found. */
127 return 1; /* Ok, no conflict. */
130 /* Fuse the array base of colocated arrays. */
131 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
134 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
135 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
136 return (int32_t)sizeof(GCtab
);
140 /* Fuse array/hash/upvalue reference into register+offset operand. */
141 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
144 if (ra_noreg(ir
->r
)) {
145 if (ir
->o
== IR_AREF
) {
146 if (mayfuse(as
, ref
)) {
147 if (irref_isk(ir
->op2
)) {
148 IRRef tab
= IR(ir
->op1
)->op1
;
149 int32_t ofs
= asm_fuseabase(as
, tab
);
150 IRRef refa
= ofs
? tab
: ir
->op1
;
151 ofs
+= 8*IR(ir
->op2
)->i
;
154 return ra_alloc1(as
, refa
, allow
);
158 } else if (ir
->o
== IR_HREFK
) {
159 if (mayfuse(as
, ref
)) {
160 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
163 return ra_alloc1(as
, ir
->op1
, allow
);
166 } else if (ir
->o
== IR_UREFC
) {
167 if (irref_isk(ir
->op1
)) {
168 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
169 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
170 int32_t jgl
= (intptr_t)J2G(as
->J
);
171 if ((uint32_t)(ofs
-jgl
) < 65536) {
172 *ofsp
= ofs
-jgl
-32768;
175 *ofsp
= (int16_t)ofs
;
176 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
182 return ra_alloc1(as
, ref
, allow
);
185 /* Fuse XLOAD/XSTORE reference into load/store operand. */
186 static void asm_fusexref(ASMState
*as
, MIPSIns mi
, Reg rt
, IRRef ref
,
187 RegSet allow
, int32_t ofs
)
191 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
192 if (ir
->o
== IR_ADD
) {
194 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
198 } else if (ir
->o
== IR_STRREF
) {
199 int32_t ofs2
= 65536;
200 lua_assert(ofs
== 0);
201 ofs
= (int32_t)sizeof(GCstr
);
202 if (irref_isk(ir
->op2
)) {
203 ofs2
= ofs
+ IR(ir
->op2
)->i
;
205 } else if (irref_isk(ir
->op1
)) {
206 ofs2
= ofs
+ IR(ir
->op1
)->i
;
209 if (!checki16(ofs2
)) {
210 /* NYI: Fuse ADD with constant. */
211 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
212 right
= (left
>> 8); left
&= 255;
213 emit_hsi(as
, mi
, rt
, RID_TMP
, ofs
);
214 emit_dst(as
, MIPSI_ADDU
, RID_TMP
, left
, right
);
220 base
= ra_alloc1(as
, ref
, allow
);
221 emit_hsi(as
, mi
, rt
, base
, ofs
);
224 /* -- Calls --------------------------------------------------------------- */
226 /* Generate a call to a C function. */
227 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
229 uint32_t n
, nargs
= CCI_NARGS(ci
);
231 Reg gpr
, fpr
= REGARG_FIRSTFPR
;
232 if ((void *)ci
->func
)
233 emit_call(as
, (void *)ci
->func
);
234 for (gpr
= REGARG_FIRSTGPR
; gpr
<= REGARG_LASTGPR
; gpr
++)
235 as
->cost
[gpr
] = REGCOST(~0u, ASMREF_L
);
236 gpr
= REGARG_FIRSTGPR
;
237 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
241 if (irt_isfp(ir
->t
) && fpr
<= REGARG_LASTFPR
&&
242 !(ci
->flags
& CCI_VARARG
)) {
243 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
244 ra_leftov(as
, fpr
, ref
);
246 gpr
+= irt_isnum(ir
->t
) ? 2 : 1;
248 fpr
= REGARG_LASTFPR
+1;
249 if (irt_isnum(ir
->t
)) gpr
= (gpr
+1) & ~1;
250 if (gpr
<= REGARG_LASTGPR
) {
251 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
252 if (irt_isfp(ir
->t
)) {
253 RegSet of
= as
->freeset
;
255 /* Workaround to protect argument GPRs from being used for remat. */
256 as
->freeset
&= ~RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1);
257 r
= ra_alloc1(as
, ref
, RSET_FPR
);
258 as
->freeset
|= (of
& RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1));
259 if (irt_isnum(ir
->t
)) {
260 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?0:1), r
+1);
261 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?1:0), r
);
262 lua_assert(rset_test(as
->freeset
, gpr
+1)); /* Already evicted. */
264 } else if (irt_isfloat(ir
->t
)) {
265 emit_tg(as
, MIPSI_MFC1
, gpr
, r
);
269 ra_leftov(as
, gpr
, ref
);
273 Reg r
= ra_alloc1z(as
, ref
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
274 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
275 emit_spstore(as
, ir
, r
, ofs
);
276 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
280 fpr
= REGARG_LASTFPR
+1;
281 if (gpr
<= REGARG_LASTGPR
)
289 /* Setup result reg/sp for call. Evict scratch regs. */
290 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
292 RegSet drop
= RSET_SCRATCH
;
293 int hiop
= ((ir
+1)->o
== IR_HIOP
);
294 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
296 if (ra_hasreg(ir
->r
))
297 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
298 if (hiop
&& ra_hasreg((ir
+1)->r
))
299 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
300 ra_evictset(as
, drop
); /* Evictions must be performed first. */
302 lua_assert(!irt_ispri(ir
->t
));
303 if (irt_isfp(ir
->t
)) {
304 if ((ci
->flags
& CCI_CASTU64
)) {
305 int32_t ofs
= sps_scale(ir
->s
);
307 if (ra_hasreg(dest
)) {
309 ra_modified(as
, dest
);
310 emit_tg(as
, MIPSI_MTC1
, RID_RETHI
, dest
+1);
311 emit_tg(as
, MIPSI_MTC1
, RID_RETLO
, dest
);
314 emit_tsi(as
, MIPSI_SW
, RID_RETLO
, RID_SP
, ofs
+(LJ_BE
?4:0));
315 emit_tsi(as
, MIPSI_SW
, RID_RETHI
, RID_SP
, ofs
+(LJ_BE
?0:4));
318 ra_destreg(as
, ir
, RID_FPRET
);
323 ra_destreg(as
, ir
, RID_RET
);
328 static void asm_call(ASMState
*as
, IRIns
*ir
)
330 IRRef args
[CCI_NARGS_MAX
];
331 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
332 asm_collectargs(as
, ir
, ci
, args
);
333 asm_setupresult(as
, ir
, ci
);
334 asm_gencall(as
, ci
, args
);
337 static void asm_callx(ASMState
*as
, IRIns
*ir
)
339 IRRef args
[CCI_NARGS_MAX
];
343 ci
.flags
= asm_callx_flags(as
, ir
);
344 asm_collectargs(as
, ir
, &ci
, args
);
345 asm_setupresult(as
, ir
, &ci
);
346 func
= ir
->op2
; irf
= IR(func
);
347 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
348 if (irref_isk(func
)) { /* Call to constant address. */
349 ci
.func
= (ASMFunction
)(void *)(irf
->i
);
350 } else { /* Need specific register for indirect calls. */
351 Reg r
= ra_alloc1(as
, func
, RID2RSET(RID_CFUNCADDR
));
353 if (r
== RID_CFUNCADDR
)
356 *--p
= MIPSI_MOVE
| MIPSF_D(RID_CFUNCADDR
) | MIPSF_S(r
);
357 *--p
= MIPSI_JALR
| MIPSF_S(r
);
359 ci
.func
= (ASMFunction
)(void *)0;
361 asm_gencall(as
, &ci
, args
);
364 static void asm_callid(ASMState
*as
, IRIns
*ir
, IRCallID id
)
366 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
370 asm_setupresult(as
, ir
, ci
);
371 asm_gencall(as
, ci
, args
);
374 static void asm_callround(ASMState
*as
, IRIns
*ir
, IRCallID id
)
376 /* The modified regs must match with the *.dasc implementation. */
377 RegSet drop
= RID2RSET(RID_R1
)|RID2RSET(RID_R12
)|RID2RSET(RID_FPRET
)|
378 RID2RSET(RID_F2
)|RID2RSET(RID_F4
)|RID2RSET(REGARG_FIRSTFPR
);
379 if (ra_hasreg(ir
->r
)) rset_clear(drop
, ir
->r
);
380 ra_evictset(as
, drop
);
381 ra_destreg(as
, ir
, RID_FPRET
);
382 emit_call(as
, (void *)lj_ir_callinfo
[id
].func
);
383 ra_leftov(as
, REGARG_FIRSTFPR
, ir
->op1
);
386 /* -- Returns ------------------------------------------------------------- */
388 /* Return to lower frame. Guard that it goes to the right spot. */
389 static void asm_retf(ASMState
*as
, IRIns
*ir
)
391 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
392 void *pc
= ir_kptr(IR(ir
->op2
));
393 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
394 as
->topslot
-= (BCReg
)delta
;
395 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
396 emit_setgl(as
, base
, jit_base
);
397 emit_addptr(as
, base
, -8*delta
);
398 asm_guard(as
, MIPSI_BNE
, RID_TMP
,
399 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
400 emit_tsi(as
, MIPSI_LW
, RID_TMP
, base
, -8);
403 /* -- Type conversions ---------------------------------------------------- */
405 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
407 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
408 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
409 asm_guard(as
, MIPSI_BC1F
, 0, 0);
410 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmp
, left
);
411 emit_fg(as
, MIPSI_CVT_D_W
, tmp
, tmp
);
412 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
413 emit_fg(as
, MIPSI_CVT_W_D
, tmp
, left
);
416 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
418 RegSet allow
= RSET_FPR
;
419 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
420 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
421 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
422 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
423 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
424 emit_fgh(as
, MIPSI_ADD_D
, tmp
, left
, right
);
427 static void asm_conv(ASMState
*as
, IRIns
*ir
)
429 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
430 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
431 IRRef lref
= ir
->op1
;
432 lua_assert(irt_type(ir
->t
) != st
);
433 lua_assert(!(irt_isint64(ir
->t
) ||
434 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
435 if (irt_isfp(ir
->t
)) {
436 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
437 if (stfp
) { /* FP to FP conversion. */
438 emit_fg(as
, st
== IRT_NUM
? MIPSI_CVT_S_D
: MIPSI_CVT_D_S
,
439 dest
, ra_alloc1(as
, lref
, RSET_FPR
));
440 } else if (st
== IRT_U32
) { /* U32 to FP conversion. */
441 /* y = (x ^ 0x8000000) + 2147483648.0 */
442 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
443 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
444 emit_fgh(as
, irt_isfloat(ir
->t
) ? MIPSI_ADD_S
: MIPSI_ADD_D
,
446 emit_fg(as
, irt_isfloat(ir
->t
) ? MIPSI_CVT_S_W
: MIPSI_CVT_D_W
,
448 if (irt_isfloat(ir
->t
))
449 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
450 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,4f000000
)),
453 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
454 (void *)lj_ir_k64_find(as
->J
, U64x(41e00000
,00000000)),
456 emit_tg(as
, MIPSI_MTC1
, RID_TMP
, dest
);
457 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, left
);
458 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
459 } else { /* Integer to FP conversion. */
460 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
461 emit_fg(as
, irt_isfloat(ir
->t
) ? MIPSI_CVT_S_W
: MIPSI_CVT_D_W
,
463 emit_tg(as
, MIPSI_MTC1
, left
, dest
);
465 } else if (stfp
) { /* FP to integer conversion. */
466 if (irt_isguard(ir
->t
)) {
467 /* Checked conversions are only supported from number to int. */
468 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
469 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
471 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
472 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
473 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
474 if (irt_isu32(ir
->t
)) {
475 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
476 emit_dst(as
, MIPSI_XOR
, dest
, dest
, RID_TMP
);
477 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
478 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
479 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_FLOOR_W_S
: MIPSI_FLOOR_W_D
,
481 emit_fgh(as
, st
== IRT_FLOAT
? MIPSI_SUB_S
: MIPSI_SUB_D
,
484 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
485 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,4f000000
)),
488 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
489 (void *)lj_ir_k64_find(as
->J
, U64x(41e00000
,00000000)),
492 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
493 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_TRUNC_W_S
: MIPSI_TRUNC_W_D
,
498 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
499 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
500 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
501 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
502 if ((ir
->op2
& IRCONV_SEXT
)) {
503 if ((as
->flags
& JIT_F_MIPS32R2
)) {
504 emit_dst(as
, st
== IRT_I8
? MIPSI_SEB
: MIPSI_SEH
, dest
, 0, left
);
506 uint32_t shift
= st
== IRT_I8
? 24 : 16;
507 emit_dta(as
, MIPSI_SRA
, dest
, dest
, shift
);
508 emit_dta(as
, MIPSI_SLL
, dest
, left
, shift
);
511 emit_tsi(as
, MIPSI_ANDI
, dest
, left
,
512 (int32_t)(st
== IRT_U8
? 0xff : 0xffff));
514 } else { /* 32/64 bit integer conversions. */
515 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
516 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
522 static void asm_conv64(ASMState
*as
, IRIns
*ir
)
524 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
525 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
529 args
[LJ_BE
?0:1] = ir
->op1
;
530 args
[LJ_BE
?1:0] = (ir
-1)->op1
;
531 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) {
532 id
= IRCALL_fp64_d2l
+ ((st
== IRT_FLOAT
) ? 2 : 0) + (dt
- IRT_I64
);
535 id
= IRCALL_fp64_l2d
+ ((dt
== IRT_FLOAT
) ? 2 : 0) + (st
- IRT_I64
);
537 ci
= &lj_ir_callinfo
[id
];
538 asm_setupresult(as
, ir
, ci
);
539 asm_gencall(as
, ci
, args
);
543 static void asm_strto(ASMState
*as
, IRIns
*ir
)
545 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
547 RegSet drop
= RSET_SCRATCH
;
548 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
549 ra_evictset(as
, drop
);
550 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_ZERO
); /* Test return status. */
551 args
[0] = ir
->op1
; /* GCstr *str */
552 args
[1] = ASMREF_TMP1
; /* TValue *n */
553 asm_gencall(as
, ci
, args
);
554 /* Store the result to the spill slot or temp slots. */
555 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
),
556 RID_SP
, sps_scale(ir
->s
));
559 /* Get pointer to TValue. */
560 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
563 if (irt_isnum(ir
->t
)) {
564 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
565 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
566 else /* Otherwise force a spill and use the spill slot. */
567 emit_tsi(as
, MIPSI_ADDIU
, dest
, RID_SP
, ra_spill(as
, ir
));
569 /* Otherwise use g->tmptv to hold the TValue. */
570 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
572 emit_tsi(as
, MIPSI_ADDIU
, dest
, RID_JGL
, offsetof(global_State
, tmptv
)-32768);
573 if (!irt_ispri(ir
->t
)) {
574 Reg src
= ra_alloc1(as
, ref
, allow
);
575 emit_setgl(as
, src
, tmptv
.gcr
);
577 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
578 emit_setgl(as
, type
, tmptv
.it
);
582 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
587 if (irt_isnum(IR(ir
->op1
)->t
) || (ir
+1)->o
== IR_HIOP
) {
588 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
589 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
590 asm_setupresult(as
, ir
, ci
); /* GCstr * */
591 asm_gencall(as
, ci
, args
);
592 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op1
);
594 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
595 args
[1] = ir
->op1
; /* int32_t k */
596 asm_setupresult(as
, ir
, ci
); /* GCstr * */
597 asm_gencall(as
, ci
, args
);
601 /* -- Memory references --------------------------------------------------- */
603 static void asm_aref(ASMState
*as
, IRIns
*ir
)
605 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
607 if (irref_isk(ir
->op2
)) {
608 IRRef tab
= IR(ir
->op1
)->op1
;
609 int32_t ofs
= asm_fuseabase(as
, tab
);
610 IRRef refa
= ofs
? tab
: ir
->op1
;
611 ofs
+= 8*IR(ir
->op2
)->i
;
613 base
= ra_alloc1(as
, refa
, RSET_GPR
);
614 emit_tsi(as
, MIPSI_ADDIU
, dest
, base
, ofs
);
618 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
619 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
620 emit_dst(as
, MIPSI_ADDU
, dest
, RID_TMP
, base
);
621 emit_dta(as
, MIPSI_SLL
, RID_TMP
, idx
, 3);
624 /* Inlined hash lookup. Specialized for key type and for const keys.
625 ** The equivalent C code is:
626 ** Node *n = hashkey(t, key);
628 ** if (lj_obj_equal(&n->key, key)) return &n->val;
629 ** } while ((n = nextnode(n)));
632 static void asm_href(ASMState
*as
, IRIns
*ir
)
634 RegSet allow
= RSET_GPR
;
635 int destused
= ra_used(ir
);
636 Reg dest
= ra_dest(as
, ir
, allow
);
637 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
638 Reg key
= RID_NONE
, type
= RID_NONE
, tmpnum
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
639 IRRef refkey
= ir
->op2
;
640 IRIns
*irkey
= IR(refkey
);
641 IRType1 kt
= irkey
->t
;
643 MCLabel l_end
, l_loop
, l_next
;
645 rset_clear(allow
, tab
);
647 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
648 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
649 } else if (!irt_ispri(kt
)) {
650 key
= ra_alloc1(as
, refkey
, allow
);
651 rset_clear(allow
, key
);
652 type
= ra_allock(as
, irt_toitype(irkey
->t
), allow
);
653 rset_clear(allow
, type
);
655 tmp2
= ra_scratch(as
, allow
);
656 rset_clear(allow
, tmp2
);
658 /* Key not found in chain: load niltv. */
659 l_end
= emit_label(as
);
661 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
663 *--as
->mcp
= MIPSI_NOP
;
664 /* Follow hash chain until the end. */
665 emit_move(as
, dest
, tmp1
);
667 emit_tsi(as
, MIPSI_LW
, tmp1
, dest
, (int32_t)offsetof(Node
, next
));
668 l_next
= emit_label(as
);
670 /* Type and value comparison. */
672 emit_branch(as
, MIPSI_BC1T
, 0, 0, l_end
);
673 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmpnum
, key
);
674 emit_tg(as
, MIPSI_MFC1
, tmp1
, key
+1);
675 emit_branch(as
, MIPSI_BEQ
, tmp1
, RID_ZERO
, l_next
);
676 emit_tsi(as
, MIPSI_SLTIU
, tmp1
, tmp1
, (int32_t)LJ_TISNUM
);
677 emit_hsi(as
, MIPSI_LDC1
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
680 emit_branch(as
, MIPSI_BEQ
, tmp1
, type
, l_end
);
682 emit_branch(as
, MIPSI_BEQ
, tmp2
, key
, l_end
);
683 emit_tsi(as
, MIPSI_LW
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
684 emit_branch(as
, MIPSI_BNE
, tmp1
, type
, l_next
);
687 emit_tsi(as
, MIPSI_LW
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
688 *l_loop
= MIPSI_BNE
| MIPSF_S(tmp1
) | ((as
->mcp
-l_loop
-1) & 0xffffu
);
690 /* Load main position relative to tab->node into dest. */
691 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
693 emit_tsi(as
, MIPSI_LW
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
696 if (irref_isk(refkey
))
697 tmphash
= ra_allock(as
, khash
, allow
);
698 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, tmp1
);
699 lua_assert(sizeof(Node
) == 24);
700 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp2
, tmp1
);
701 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, 3);
702 emit_dta(as
, MIPSI_SLL
, tmp2
, tmp1
, 5);
703 emit_dst(as
, MIPSI_AND
, tmp1
, tmp2
, tmphash
);
704 emit_tsi(as
, MIPSI_LW
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
705 emit_tsi(as
, MIPSI_LW
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
706 if (irref_isk(refkey
)) {
708 } else if (irt_isstr(kt
)) {
709 emit_tsi(as
, MIPSI_LW
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
710 } else { /* Must match with hash*() in lj_tab.c. */
711 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp1
, tmp2
);
712 emit_rotr(as
, tmp2
, tmp2
, dest
, (-HASH_ROT3
)&31);
713 emit_dst(as
, MIPSI_XOR
, tmp1
, tmp1
, tmp2
);
714 emit_rotr(as
, tmp1
, tmp1
, dest
, (-HASH_ROT2
-HASH_ROT1
)&31);
715 emit_dst(as
, MIPSI_SUBU
, tmp2
, tmp2
, dest
);
717 emit_dst(as
, MIPSI_XOR
, tmp2
, tmp2
, tmp1
);
718 if ((as
->flags
& JIT_F_MIPS32R2
)) {
719 emit_dta(as
, MIPSI_ROTR
, dest
, tmp1
, (-HASH_ROT1
)&31);
721 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp1
);
722 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, HASH_ROT1
);
723 emit_dta(as
, MIPSI_SRL
, dest
, tmp1
, (-HASH_ROT1
)&31);
725 emit_dst(as
, MIPSI_ADDU
, tmp1
, tmp1
, tmp1
);
726 emit_tg(as
, MIPSI_MFC1
, tmp2
, key
);
727 emit_tg(as
, MIPSI_MFC1
, tmp1
, key
+1);
729 emit_dst(as
, MIPSI_XOR
, tmp2
, key
, tmp1
);
730 emit_rotr(as
, dest
, tmp1
, tmp2
, (-HASH_ROT1
)&31);
731 emit_dst(as
, MIPSI_ADDU
, tmp1
, key
, ra_allock(as
, HASH_BIAS
, allow
));
737 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
739 IRIns
*kslot
= IR(ir
->op2
);
740 IRIns
*irkey
= IR(kslot
->op1
);
741 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
742 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
743 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
744 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
745 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
746 RegSet allow
= rset_exclude(RSET_GPR
, node
);
748 lua_assert(ofs
% sizeof(Node
) == 0);
751 rset_clear(allow
, dest
);
752 kofs
= (int32_t)offsetof(Node
, key
);
753 } else if (ra_hasreg(dest
)) {
754 emit_tsi(as
, MIPSI_ADDIU
, dest
, node
, ofs
);
756 if (!irt_ispri(irkey
->t
)) {
757 key
= ra_scratch(as
, allow
);
758 rset_clear(allow
, key
);
760 if (irt_isnum(irkey
->t
)) {
761 lo
= (int32_t)ir_knum(irkey
)->u32
.lo
;
762 hi
= (int32_t)ir_knum(irkey
)->u32
.hi
;
765 hi
= irt_toitype(irkey
->t
);
769 asm_guard(as
, MIPSI_BNE
, key
, lo
? ra_allock(as
, lo
, allow
) : RID_ZERO
);
771 asm_guard(as
, MIPSI_BNE
, type
, hi
? ra_allock(as
, hi
, allow
) : RID_ZERO
);
772 if (ra_hasreg(key
)) emit_tsi(as
, MIPSI_LW
, key
, idx
, kofs
+(LJ_BE
?4:0));
773 emit_tsi(as
, MIPSI_LW
, type
, idx
, kofs
+(LJ_BE
?0:4));
775 emit_tsi(as
, MIPSI_ADDU
, dest
, node
, ra_allock(as
, ofs
, allow
));
778 static void asm_newref(ASMState
*as
, IRIns
*ir
)
780 if (ir
->r
!= RID_SINK
) {
781 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
783 args
[0] = ASMREF_L
; /* lua_State *L */
784 args
[1] = ir
->op1
; /* GCtab *t */
785 args
[2] = ASMREF_TMP1
; /* cTValue *key */
786 asm_setupresult(as
, ir
, ci
); /* TValue * */
787 asm_gencall(as
, ci
, args
);
788 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op2
);
792 static void asm_uref(ASMState
*as
, IRIns
*ir
)
794 /* NYI: Check that UREFO is still open and not aliasing a slot. */
795 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
796 if (irref_isk(ir
->op1
)) {
797 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
798 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
799 emit_lsptr(as
, MIPSI_LW
, dest
, v
, RSET_GPR
);
801 Reg uv
= ra_scratch(as
, RSET_GPR
);
802 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
803 if (ir
->o
== IR_UREFC
) {
804 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
805 emit_tsi(as
, MIPSI_ADDIU
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
806 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
808 emit_tsi(as
, MIPSI_LW
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
810 emit_tsi(as
, MIPSI_LW
, uv
, func
,
811 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
815 static void asm_fref(ASMState
*as
, IRIns
*ir
)
817 UNUSED(as
); UNUSED(ir
);
818 lua_assert(!ra_used(ir
));
821 static void asm_strref(ASMState
*as
, IRIns
*ir
)
823 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
824 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
825 int32_t ofs
= (int32_t)sizeof(GCstr
);
827 if (irref_isk(ref
)) {
828 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
829 } else if (!irref_isk(refk
)) {
830 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
831 IRIns
*irr
= IR(ir
->op2
);
832 if (ra_hasreg(irr
->r
)) {
833 ra_noweak(as
, irr
->r
);
835 } else if (mayfuse(as
, irr
->op2
) &&
836 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
837 checki16(ofs
+ IR(irr
->op2
)->i
)) {
838 ofs
+= IR(irr
->op2
)->i
;
839 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
841 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
843 emit_tsi(as
, MIPSI_ADDIU
, dest
, dest
, ofs
);
844 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
847 r
= ra_alloc1(as
, ref
, RSET_GPR
);
850 emit_tsi(as
, MIPSI_ADDIU
, dest
, r
, ofs
);
852 emit_dst(as
, MIPSI_ADDU
, dest
, r
,
853 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
856 /* -- Loads and stores ---------------------------------------------------- */
858 static MIPSIns
asm_fxloadins(IRIns
*ir
)
860 switch (irt_type(ir
->t
)) {
861 case IRT_I8
: return MIPSI_LB
;
862 case IRT_U8
: return MIPSI_LBU
;
863 case IRT_I16
: return MIPSI_LH
;
864 case IRT_U16
: return MIPSI_LHU
;
865 case IRT_NUM
: return MIPSI_LDC1
;
866 case IRT_FLOAT
: return MIPSI_LWC1
;
867 default: return MIPSI_LW
;
871 static MIPSIns
asm_fxstoreins(IRIns
*ir
)
873 switch (irt_type(ir
->t
)) {
874 case IRT_I8
: case IRT_U8
: return MIPSI_SB
;
875 case IRT_I16
: case IRT_U16
: return MIPSI_SH
;
876 case IRT_NUM
: return MIPSI_SDC1
;
877 case IRT_FLOAT
: return MIPSI_SWC1
;
878 default: return MIPSI_SW
;
882 static void asm_fload(ASMState
*as
, IRIns
*ir
)
884 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
885 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
886 MIPSIns mi
= asm_fxloadins(ir
);
888 if (ir
->op2
== IRFL_TAB_ARRAY
) {
889 ofs
= asm_fuseabase(as
, ir
->op1
);
890 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
891 emit_tsi(as
, MIPSI_ADDIU
, dest
, idx
, ofs
);
895 ofs
= field_ofs
[ir
->op2
];
896 lua_assert(!irt_isfp(ir
->t
));
897 emit_tsi(as
, mi
, dest
, idx
, ofs
);
900 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
902 if (ir
->r
!= RID_SINK
) {
903 Reg src
= ra_alloc1z(as
, ir
->op2
, RSET_GPR
);
904 IRIns
*irf
= IR(ir
->op1
);
905 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
906 int32_t ofs
= field_ofs
[irf
->op2
];
907 MIPSIns mi
= asm_fxstoreins(ir
);
908 lua_assert(!irt_isfp(ir
->t
));
909 emit_tsi(as
, mi
, src
, idx
, ofs
);
913 static void asm_xload(ASMState
*as
, IRIns
*ir
)
915 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
916 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
917 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
920 static void asm_xstore(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
922 if (ir
->r
!= RID_SINK
) {
923 Reg src
= ra_alloc1z(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
924 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
925 rset_exclude(RSET_GPR
, src
), ofs
);
929 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
932 Reg dest
= RID_NONE
, type
= RID_TMP
, idx
;
933 RegSet allow
= RSET_GPR
;
936 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
937 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
938 rset_clear(allow
, dest
);
940 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
941 rset_clear(allow
, idx
);
943 asm_guard(as
, MIPSI_BEQ
, type
, RID_ZERO
);
944 emit_tsi(as
, MIPSI_SLTIU
, type
, type
, (int32_t)LJ_TISNUM
);
946 emit_hsi(as
, MIPSI_LDC1
, dest
, idx
, ofs
);
948 asm_guard(as
, MIPSI_BNE
, type
, ra_allock(as
, irt_toitype(t
), allow
));
949 if (ra_hasreg(dest
)) emit_tsi(as
, MIPSI_LW
, dest
, idx
, ofs
+(LJ_BE
?4:0));
951 emit_tsi(as
, MIPSI_LW
, type
, idx
, ofs
+(LJ_BE
?0:4));
954 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
956 RegSet allow
= RSET_GPR
;
957 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
959 if (ir
->r
== RID_SINK
)
961 if (irt_isnum(ir
->t
)) {
962 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
964 if (!irt_ispri(ir
->t
)) {
965 src
= ra_alloc1(as
, ir
->op2
, allow
);
966 rset_clear(allow
, src
);
968 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
969 rset_clear(allow
, type
);
971 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
972 if (irt_isnum(ir
->t
)) {
973 emit_hsi(as
, MIPSI_SDC1
, src
, idx
, ofs
);
976 emit_tsi(as
, MIPSI_SW
, src
, idx
, ofs
+(LJ_BE
?4:0));
977 emit_tsi(as
, MIPSI_SW
, type
, idx
, ofs
+(LJ_BE
?0:4));
981 static void asm_sload(ASMState
*as
, IRIns
*ir
)
983 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 4 : 0);
985 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
986 RegSet allow
= RSET_GPR
;
987 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
988 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
989 lua_assert(!irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
990 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
991 dest
= ra_scratch(as
, RSET_FPR
);
992 asm_tointg(as
, ir
, dest
);
993 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
994 } else if (ra_used(ir
)) {
995 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
996 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
997 rset_clear(allow
, dest
);
998 base
= ra_alloc1(as
, REF_BASE
, allow
);
999 rset_clear(allow
, base
);
1000 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1002 Reg tmp
= ra_scratch(as
, RSET_FPR
);
1003 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
1004 emit_fg(as
, MIPSI_CVT_W_D
, tmp
, tmp
);
1006 t
.irt
= IRT_NUM
; /* Check for original type. */
1008 Reg tmp
= ra_scratch(as
, RSET_GPR
);
1009 emit_fg(as
, MIPSI_CVT_D_W
, dest
, dest
);
1010 emit_tg(as
, MIPSI_MTC1
, tmp
, dest
);
1012 t
.irt
= IRT_INT
; /* Check for original type. */
1017 base
= ra_alloc1(as
, REF_BASE
, allow
);
1018 rset_clear(allow
, base
);
1021 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1022 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1023 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)LJ_TISNUM
);
1026 if (ra_hasreg(dest
)) emit_hsi(as
, MIPSI_LDC1
, dest
, base
, ofs
);
1028 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1029 Reg ktype
= ra_allock(as
, irt_toitype(t
), allow
);
1030 asm_guard(as
, MIPSI_BNE
, RID_TMP
, ktype
);
1033 if (ra_hasreg(dest
)) emit_tsi(as
, MIPSI_LW
, dest
, base
, ofs
^ (LJ_BE
?4:0));
1035 if (ra_hasreg(type
)) emit_tsi(as
, MIPSI_LW
, type
, base
, ofs
^ (LJ_BE
?0:4));
1038 /* -- Allocations --------------------------------------------------------- */
1041 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1043 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1044 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1045 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1046 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1047 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1049 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1050 RegSet drop
= RSET_SCRATCH
;
1051 lua_assert(sz
!= CTSIZE_INVALID
);
1053 args
[0] = ASMREF_L
; /* lua_State *L */
1054 args
[1] = ASMREF_TMP1
; /* MSize size */
1057 if (ra_hasreg(ir
->r
))
1058 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1059 ra_evictset(as
, drop
);
1061 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1063 /* Initialize immutable cdata object. */
1064 if (ir
->o
== IR_CNEWI
) {
1065 int32_t ofs
= sizeof(GCcdata
);
1066 lua_assert(sz
== 4 || sz
== 8);
1069 lua_assert((ir
+1)->o
== IR_HIOP
);
1073 Reg r
= ra_alloc1z(as
, ir
->op2
, allow
);
1074 emit_tsi(as
, MIPSI_SW
, r
, RID_RET
, ofs
);
1075 rset_clear(allow
, r
);
1076 if (ofs
== sizeof(GCcdata
)) break;
1077 ofs
-= 4; if (LJ_BE
) ir
++; else ir
--;
1080 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1081 emit_tsi(as
, MIPSI_SB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1082 emit_tsi(as
, MIPSI_SH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1083 emit_ti(as
, MIPSI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1084 emit_ti(as
, MIPSI_LI
, RID_TMP
, ctypeid
); /* Lower 16 bit used. Sign-ext ok. */
1085 asm_gencall(as
, ci
, args
);
1086 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1087 ra_releasetmp(as
, ASMREF_TMP1
));
1090 #define asm_cnew(as, ir) ((void)0)
1093 /* -- Write barriers ------------------------------------------------------ */
1095 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1097 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1098 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1100 MCLabel l_end
= emit_label(as
);
1101 emit_tsi(as
, MIPSI_SW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1102 emit_tsi(as
, MIPSI_SB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1103 emit_setgl(as
, tab
, gc
.grayagain
);
1104 emit_getgl(as
, link
, gc
.grayagain
);
1105 emit_dst(as
, MIPSI_XOR
, mark
, mark
, RID_TMP
); /* Clear black bit. */
1106 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1107 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, mark
, LJ_GC_BLACK
);
1108 emit_tsi(as
, MIPSI_LBU
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1111 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1113 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1117 /* No need for other object barriers (yet). */
1118 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1119 ra_evictset(as
, RSET_SCRATCH
);
1120 l_end
= emit_label(as
);
1121 args
[0] = ASMREF_TMP1
; /* global_State *g */
1122 args
[1] = ir
->op1
; /* TValue *tv */
1123 asm_gencall(as
, ci
, args
);
1124 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1125 obj
= IR(ir
->op1
)->r
;
1126 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1127 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1128 emit_tsi(as
, MIPSI_ANDI
, tmp
, tmp
, LJ_GC_BLACK
);
1129 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1130 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1131 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1132 emit_tsi(as
, MIPSI_LBU
, tmp
, obj
,
1133 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1134 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1137 /* -- Arithmetic and logic operations ------------------------------------- */
1139 static void asm_fparith(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1141 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1142 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1143 right
= (left
>> 8); left
&= 255;
1144 emit_fgh(as
, mi
, dest
, left
, right
);
1147 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1149 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1150 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1151 emit_fg(as
, mi
, dest
, left
);
1154 static int asm_fpjoin_pow(ASMState
*as
, IRIns
*ir
)
1156 IRIns
*irp
= IR(ir
->op1
);
1157 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1158 IRIns
*irpp
= IR(irp
->op1
);
1159 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1160 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1161 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_pow
];
1163 args
[0] = irpp
->op1
;
1165 asm_setupresult(as
, ir
, ci
);
1166 asm_gencall(as
, ci
, args
);
1173 static void asm_add(ASMState
*as
, IRIns
*ir
)
1175 if (irt_isnum(ir
->t
)) {
1176 asm_fparith(as
, ir
, MIPSI_ADD_D
);
1178 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1179 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1180 if (irref_isk(ir
->op2
)) {
1181 int32_t k
= IR(ir
->op2
)->i
;
1183 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1187 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1188 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1192 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1194 if (irt_isnum(ir
->t
)) {
1195 asm_fparith(as
, ir
, MIPSI_SUB_D
);
1197 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1198 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1199 right
= (left
>> 8); left
&= 255;
1200 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1204 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1206 if (irt_isnum(ir
->t
)) {
1207 asm_fparith(as
, ir
, MIPSI_MUL_D
);
1209 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1210 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1211 right
= (left
>> 8); left
&= 255;
1212 emit_dst(as
, MIPSI_MUL
, dest
, left
, right
);
1216 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1218 if (irt_isnum(ir
->t
)) {
1219 asm_fpunary(as
, ir
, MIPSI_NEG_D
);
1221 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1222 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1223 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1227 static void asm_arithov(ASMState
*as
, IRIns
*ir
)
1229 Reg right
, left
, tmp
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1230 if (irref_isk(ir
->op2
)) {
1231 int k
= IR(ir
->op2
)->i
;
1232 if (ir
->o
== IR_SUBOV
) k
= -k
;
1233 if (checki16(k
)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1234 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1235 asm_guard(as
, k
>= 0 ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1236 emit_dst(as
, MIPSI_SLT
, RID_TMP
, dest
, dest
== left
? RID_TMP
: left
);
1237 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1238 if (dest
== left
) emit_move(as
, RID_TMP
, left
);
1242 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1243 right
= (left
>> 8); left
&= 255;
1244 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
, left
),
1246 asm_guard(as
, MIPSI_BLTZ
, RID_TMP
, 0);
1247 emit_dst(as
, MIPSI_AND
, RID_TMP
, RID_TMP
, tmp
);
1248 if (ir
->o
== IR_ADDOV
) { /* ((dest^left) & (dest^right)) < 0 */
1249 emit_dst(as
, MIPSI_XOR
, RID_TMP
, dest
, dest
== right
? RID_TMP
: right
);
1250 } else { /* ((dest^left) & (dest^~right)) < 0 */
1251 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, dest
);
1252 emit_dst(as
, MIPSI_NOR
, RID_TMP
, dest
== right
? RID_TMP
: right
, RID_ZERO
);
1254 emit_dst(as
, MIPSI_XOR
, tmp
, dest
, dest
== left
? RID_TMP
: left
);
1255 emit_dst(as
, ir
->o
== IR_ADDOV
? MIPSI_ADDU
: MIPSI_SUBU
, dest
, left
, right
);
1256 if (dest
== left
|| dest
== right
)
1257 emit_move(as
, RID_TMP
, dest
== left
? left
: right
);
1260 static void asm_mulov(ASMState
*as
, IRIns
*ir
)
1265 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused in single-number mode. */
1270 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1272 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1273 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1274 if (irref_isk(ir
->op2
)) {
1275 int32_t k
= IR(ir
->op2
)->i
;
1277 emit_dst(as
, MIPSI_ADDU
, dest
, left
, RID_TMP
);
1279 } else if (checki16(k
)) {
1280 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1281 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1285 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1286 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1287 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1290 dest
= ra_dest(as
, ir
, RSET_GPR
);
1291 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1292 if (irref_isk(ir
->op2
)) {
1293 int32_t k
= IR(ir
->op2
)->i
;
1296 emit_move(as
, dest
, left
);
1298 } else if (checki16(k
)) {
1300 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, left
));
1301 emit_move(as
, dest
, tmp
);
1304 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, left
);
1305 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1309 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1310 if (dest
== left
&& dest
== right
) {
1311 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1312 emit_move(as
, dest
, tmp
);
1315 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, dest
== left
? right
: left
);
1316 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1319 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1321 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1322 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1323 right
= (left
>> 8); left
&= 255;
1324 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
1325 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1327 dest
= ra_dest(as
, ir
, RSET_GPR
);
1328 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1329 right
= (left
>> 8); left
&= 255;
1331 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1332 emit_move(as
, dest
, tmp
);
1335 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, left
, dest
);
1336 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
1339 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1341 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1342 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1343 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
1344 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1346 dest
= ra_dest(as
, ir
, RSET_GPR
);
1347 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1348 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_ZERO
, dest
);
1349 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
1353 static void asm_bitnot(ASMState
*as
, IRIns
*ir
)
1355 Reg left
, right
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1356 IRIns
*irl
= IR(ir
->op1
);
1357 if (mayfuse(as
, ir
->op1
) && irl
->o
== IR_BOR
) {
1358 left
= ra_alloc2(as
, irl
, RSET_GPR
);
1359 right
= (left
>> 8); left
&= 255;
1361 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1364 emit_dst(as
, MIPSI_NOR
, dest
, left
, right
);
1367 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1369 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1370 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1371 if ((as
->flags
& JIT_F_MIPS32R2
)) {
1372 emit_dta(as
, MIPSI_ROTR
, dest
, RID_TMP
, 16);
1373 emit_dst(as
, MIPSI_WSBH
, RID_TMP
, 0, left
);
1375 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), dest
));
1376 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp
);
1377 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
1378 emit_tsi(as
, MIPSI_ANDI
, dest
, dest
, 0xff00);
1379 emit_dta(as
, MIPSI_SLL
, RID_TMP
, RID_TMP
, 8);
1380 emit_dta(as
, MIPSI_SRL
, dest
, left
, 8);
1381 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, left
, 0xff00);
1382 emit_dst(as
, MIPSI_OR
, tmp
, tmp
, RID_TMP
);
1383 emit_dta(as
, MIPSI_SRL
, tmp
, left
, 24);
1384 emit_dta(as
, MIPSI_SLL
, RID_TMP
, left
, 24);
1388 static void asm_bitop(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
1390 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1391 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1392 if (irref_isk(ir
->op2
)) {
1393 int32_t k
= IR(ir
->op2
)->i
;
1395 emit_tsi(as
, mik
, dest
, left
, k
);
1399 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1400 emit_dst(as
, mi
, dest
, left
, right
);
1403 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
1405 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1406 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1407 uint32_t shift
= (uint32_t)(IR(ir
->op2
)->i
& 31);
1408 emit_dta(as
, mik
, dest
, ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
), shift
);
1410 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1411 right
= (left
>> 8); left
&= 255;
1412 emit_dst(as
, mi
, dest
, right
, left
); /* Shift amount is in rs. */
1416 static void asm_bitror(ASMState
*as
, IRIns
*ir
)
1418 if ((as
->flags
& JIT_F_MIPS32R2
)) {
1419 asm_bitshift(as
, ir
, MIPSI_ROTRV
, MIPSI_ROTR
);
1421 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1422 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1423 uint32_t shift
= (uint32_t)(IR(ir
->op2
)->i
& 31);
1424 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1425 emit_rotr(as
, dest
, left
, RID_TMP
, shift
);
1427 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1428 right
= (left
>> 8); left
&= 255;
1429 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
1430 emit_dst(as
, MIPSI_SRLV
, dest
, right
, left
);
1431 emit_dst(as
, MIPSI_SLLV
, RID_TMP
, RID_TMP
, left
);
1432 emit_dst(as
, MIPSI_SUBU
, RID_TMP
, ra_allock(as
, 32, RSET_GPR
), right
);
1437 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1439 if (irt_isnum(ir
->t
)) {
1440 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1441 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1442 right
= (left
>> 8); left
&= 255;
1444 emit_fg(as
, MIPSI_MOVT_D
, dest
, right
);
1446 emit_fg(as
, MIPSI_MOVF_D
, dest
, left
);
1447 if (dest
!= right
) emit_fg(as
, MIPSI_MOV_D
, dest
, right
);
1449 emit_fgh(as
, MIPSI_C_OLT_D
, 0, ismax
? left
: right
, ismax
? right
: left
);
1451 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1452 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1453 right
= (left
>> 8); left
&= 255;
1455 emit_dst(as
, MIPSI_MOVN
, dest
, right
, RID_TMP
);
1457 emit_dst(as
, MIPSI_MOVZ
, dest
, left
, RID_TMP
);
1458 if (dest
!= right
) emit_move(as
, dest
, right
);
1460 emit_dst(as
, MIPSI_SLT
, RID_TMP
,
1461 ismax
? left
: right
, ismax
? right
: left
);
1465 /* -- Comparisons --------------------------------------------------------- */
1467 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1469 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1471 if (irt_isnum(ir
->t
)) {
1472 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1473 right
= (left
>> 8); left
&= 255;
1474 asm_guard(as
, (op
&1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
1475 emit_fgh(as
, MIPSI_C_OLT_D
+ ((op
&3) ^ ((op
>>2)&1)), 0, left
, right
);
1477 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1478 if (op
== IR_ABC
) op
= IR_UGT
;
1479 if ((op
&4) == 0 && irref_isk(ir
->op2
) && IR(ir
->op2
)->i
== 0) {
1480 MIPSIns mi
= (op
&2) ? ((op
&1) ? MIPSI_BLEZ
: MIPSI_BGTZ
) :
1481 ((op
&1) ? MIPSI_BLTZ
: MIPSI_BGEZ
);
1482 asm_guard(as
, mi
, left
, 0);
1484 if (irref_isk(ir
->op2
)) {
1485 int32_t k
= IR(ir
->op2
)->i
;
1488 asm_guard(as
, (op
&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1489 emit_tsi(as
, (op
&4) ? MIPSI_SLTIU
: MIPSI_SLTI
,
1494 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1495 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1496 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
,
1497 RID_TMP
, (op
&2) ? right
: left
, (op
&2) ? left
: right
);
1502 static void asm_compeq(ASMState
*as
, IRIns
*ir
)
1504 Reg right
, left
= ra_alloc2(as
, ir
, irt_isnum(ir
->t
) ? RSET_FPR
: RSET_GPR
);
1505 right
= (left
>> 8); left
&= 255;
1506 if (irt_isnum(ir
->t
)) {
1507 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
1508 emit_fgh(as
, MIPSI_C_EQ_D
, 0, left
, right
);
1510 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, left
, right
);
1515 /* 64 bit integer comparisons. */
1516 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1518 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1519 IROp op
= (ir
-1)->o
;
1521 Reg rightlo
, leftlo
, righthi
, lefthi
= ra_alloc2(as
, ir
, RSET_GPR
);
1522 righthi
= (lefthi
>> 8); lefthi
&= 255;
1523 leftlo
= ra_alloc2(as
, ir
-1,
1524 rset_exclude(rset_exclude(RSET_GPR
, lefthi
), righthi
));
1525 rightlo
= (leftlo
>> 8); leftlo
&= 255;
1526 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1527 l_end
= emit_label(as
);
1528 if (lefthi
!= righthi
)
1529 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
, RID_TMP
,
1530 (op
&2) ? righthi
: lefthi
, (op
&2) ? lefthi
: righthi
);
1531 emit_dst(as
, MIPSI_SLTU
, RID_TMP
,
1532 (op
&2) ? rightlo
: leftlo
, (op
&2) ? leftlo
: rightlo
);
1533 if (lefthi
!= righthi
)
1534 emit_branch(as
, MIPSI_BEQ
, lefthi
, righthi
, l_end
);
1537 static void asm_comp64eq(ASMState
*as
, IRIns
*ir
)
1539 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1540 right
= (left
>> 8); left
&= 255;
1541 asm_guard(as
, ((ir
-1)->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, RID_TMP
, RID_ZERO
);
1542 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1543 emit_dst(as
, MIPSI_OR
, RID_TMP
, RID_TMP
, tmp
);
1544 emit_dst(as
, MIPSI_XOR
, tmp
, left
, right
);
1545 left
= ra_alloc2(as
, ir
-1, RSET_GPR
);
1546 right
= (left
>> 8); left
&= 255;
1547 emit_dst(as
, MIPSI_XOR
, RID_TMP
, left
, right
);
1551 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1553 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1554 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1557 /* HIOP is marked as a store because it needs its own DCE logic. */
1558 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1559 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1560 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1561 as
->curins
--; /* Always skip the CONV. */
1565 } else if ((ir
-1)->o
< IR_EQ
) { /* 64 bit integer comparisons. ORDER IR. */
1566 as
->curins
--; /* Always skip the loword comparison. */
1569 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1570 as
->curins
--; /* Always skip the loword comparison. */
1571 asm_comp64eq(as
, ir
);
1573 } else if ((ir
-1)->o
== IR_XSTORE
) {
1574 as
->curins
--; /* Handle both stores here. */
1575 if ((ir
-1)->r
!= RID_SINK
) {
1576 asm_xstore(as
, ir
, LJ_LE
? 4 : 0);
1577 asm_xstore(as
, ir
-1, LJ_LE
? 0 : 4);
1581 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1582 switch ((ir
-1)->o
) {
1583 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1584 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1585 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1589 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1592 /* Nothing to do here. Handled by lo op itself. */
1594 default: lua_assert(0); break;
1597 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1601 /* -- Stack handling ------------------------------------------------------ */
1603 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1604 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1605 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1607 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1608 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1609 ExitNo oldsnap
= as
->snapno
;
1610 rset_clear(allow
, pbase
);
1611 tmp
= allow
? rset_pickbot(allow
) :
1612 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1613 as
->snapno
= exitno
;
1614 asm_guard(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
);
1615 as
->snapno
= oldsnap
;
1616 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1617 emit_tsi(as
, MIPSI_LW
, tmp
, RID_SP
, 0);
1619 ra_modified(as
, tmp
);
1620 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)(8*topslot
));
1621 emit_dst(as
, MIPSI_SUBU
, RID_TMP
, tmp
, pbase
);
1622 emit_tsi(as
, MIPSI_LW
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1623 if (pbase
== RID_TMP
)
1624 emit_getgl(as
, RID_TMP
, jit_base
);
1625 emit_getgl(as
, tmp
, jit_L
);
1626 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1627 emit_tsi(as
, MIPSI_SW
, tmp
, RID_SP
, 0);
1630 /* Restore Lua stack from on-trace state. */
1631 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1633 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1634 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1635 MSize n
, nent
= snap
->nent
;
1636 /* Store the value of all modified slots to the Lua stack. */
1637 for (n
= 0; n
< nent
; n
++) {
1638 SnapEntry sn
= map
[n
];
1639 BCReg s
= snap_slot(sn
);
1640 int32_t ofs
= 8*((int32_t)s
-1);
1641 IRRef ref
= snap_ref(sn
);
1642 IRIns
*ir
= IR(ref
);
1643 if ((sn
& SNAP_NORESTORE
))
1645 if (irt_isnum(ir
->t
)) {
1646 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1647 emit_hsi(as
, MIPSI_SDC1
, src
, RID_BASE
, ofs
);
1650 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1651 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1652 if (!irt_ispri(ir
->t
)) {
1653 Reg src
= ra_alloc1(as
, ref
, allow
);
1654 rset_clear(allow
, src
);
1655 emit_tsi(as
, MIPSI_SW
, src
, RID_BASE
, ofs
+(LJ_BE
?4:0));
1657 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1658 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1659 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1661 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1663 emit_tsi(as
, MIPSI_SW
, type
, RID_BASE
, ofs
+(LJ_BE
?0:4));
1667 lua_assert(map
+ nent
== flinks
);
1670 /* -- GC handling --------------------------------------------------------- */
1672 /* Check GC threshold and do one or more GC steps. */
1673 static void asm_gc_check(ASMState
*as
)
1675 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1679 ra_evictset(as
, RSET_SCRATCH
);
1680 l_end
= emit_label(as
);
1681 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1682 /* Assumes asm_snap_prep() already done. */
1683 asm_guard(as
, MIPSI_BNE
, RID_RET
, RID_ZERO
);
1684 args
[0] = ASMREF_TMP1
; /* global_State *g */
1685 args
[1] = ASMREF_TMP2
; /* MSize steps */
1686 asm_gencall(as
, ci
, args
);
1687 emit_tsi(as
, MIPSI_ADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1688 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1689 emit_loadi(as
, tmp
, as
->gcsteps
);
1690 /* Jump around GC step if GC total < GC threshold. */
1691 emit_branch(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
, l_end
);
1692 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_TMP
, tmp
);
1693 emit_getgl(as
, tmp
, gc
.threshold
);
1694 emit_getgl(as
, RID_TMP
, gc
.total
);
1699 /* -- Loop handling ------------------------------------------------------- */
1701 /* Fixup the loop branch. */
1702 static void asm_loop_fixup(ASMState
*as
)
1704 MCode
*p
= as
->mctop
;
1705 MCode
*target
= as
->mcp
;
1707 if (as
->loopinv
) { /* Inverted loop branch? */
1708 /* asm_guard already inverted the cond branch. Only patch the target. */
1709 p
[-3] |= ((target
-p
+2) & 0x0000ffffu
);
1711 p
[-2] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1715 /* -- Head of trace ------------------------------------------------------- */
1717 /* Coalesce BASE register for a root trace. */
1718 static void asm_head_root_base(ASMState
*as
)
1720 IRIns
*ir
= IR(REF_BASE
);
1722 if (as
->loopinv
) as
->mctop
--;
1725 if (rset_test(as
->modset
, r
))
1726 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1728 emit_move(as
, r
, RID_BASE
);
1732 /* Coalesce BASE register for a side trace. */
1733 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1735 IRIns
*ir
= IR(REF_BASE
);
1737 if (as
->loopinv
) as
->mctop
--;
1740 if (rset_test(as
->modset
, r
))
1741 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1743 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1744 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1745 rset_clear(allow
, irp
->r
);
1746 emit_move(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1748 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1754 /* -- Tail of trace ------------------------------------------------------- */
1756 /* Fixup the tail code. */
1757 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1759 MCode
*target
= lnk
? traceref(as
->J
,lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1760 int32_t spadj
= as
->T
->spadjust
;
1761 MCode
*p
= as
->mctop
-1;
1762 *p
= spadj
? (MIPSI_ADDIU
|MIPSF_T(RID_SP
)|MIPSF_S(RID_SP
)|spadj
) : MIPSI_NOP
;
1763 p
[-1] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1766 /* Prepare tail of code. */
1767 static void asm_tail_prep(ASMState
*as
)
1769 as
->mcp
= as
->mctop
-2; /* Leave room for branch plus nop or stack adj. */
1770 as
->invmcp
= as
->loopref
? as
->mcp
: NULL
;
1773 /* -- Instruction dispatch ------------------------------------------------ */
1775 /* Assemble a single instruction. */
1776 static void asm_ir(ASMState
*as
, IRIns
*ir
)
1778 switch ((IROp
)ir
->o
) {
1779 /* Miscellaneous ops. */
1780 case IR_LOOP
: asm_loop(as
); break;
1781 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
1783 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
1784 case IR_PHI
: asm_phi(as
, ir
); break;
1785 case IR_HIOP
: asm_hiop(as
, ir
); break;
1786 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
1788 /* Guarded assertions. */
1789 case IR_EQ
: case IR_NE
: asm_compeq(as
, ir
); break;
1790 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
1791 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
1796 case IR_RETF
: asm_retf(as
, ir
); break;
1799 case IR_BNOT
: asm_bitnot(as
, ir
); break;
1800 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
1802 case IR_BAND
: asm_bitop(as
, ir
, MIPSI_AND
, MIPSI_ANDI
); break;
1803 case IR_BOR
: asm_bitop(as
, ir
, MIPSI_OR
, MIPSI_ORI
); break;
1804 case IR_BXOR
: asm_bitop(as
, ir
, MIPSI_XOR
, MIPSI_XORI
); break;
1806 case IR_BSHL
: asm_bitshift(as
, ir
, MIPSI_SLLV
, MIPSI_SLL
); break;
1807 case IR_BSHR
: asm_bitshift(as
, ir
, MIPSI_SRLV
, MIPSI_SRL
); break;
1808 case IR_BSAR
: asm_bitshift(as
, ir
, MIPSI_SRAV
, MIPSI_SRA
); break;
1809 case IR_BROL
: lua_assert(0); break;
1810 case IR_BROR
: asm_bitror(as
, ir
); break;
1812 /* Arithmetic ops. */
1813 case IR_ADD
: asm_add(as
, ir
); break;
1814 case IR_SUB
: asm_sub(as
, ir
); break;
1815 case IR_MUL
: asm_mul(as
, ir
); break;
1816 case IR_DIV
: asm_fparith(as
, ir
, MIPSI_DIV_D
); break;
1817 case IR_MOD
: asm_callid(as
, ir
, IRCALL_lj_vm_modi
); break;
1818 case IR_POW
: asm_callid(as
, ir
, IRCALL_lj_vm_powi
); break;
1819 case IR_NEG
: asm_neg(as
, ir
); break;
1821 case IR_ABS
: asm_fpunary(as
, ir
, MIPSI_ABS_D
); break;
1822 case IR_ATAN2
: asm_callid(as
, ir
, IRCALL_atan2
); break;
1823 case IR_LDEXP
: asm_callid(as
, ir
, IRCALL_ldexp
); break;
1824 case IR_MIN
: asm_min_max(as
, ir
, 0); break;
1825 case IR_MAX
: asm_min_max(as
, ir
, 1); break;
1827 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
1829 if (ir
->op2
<= IRFPM_TRUNC
)
1830 asm_callround(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1831 else if (ir
->op2
== IRFPM_SQRT
)
1832 asm_fpunary(as
, ir
, MIPSI_SQRT_D
);
1834 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1837 /* Overflow-checking arithmetic ops. */
1838 case IR_ADDOV
: asm_arithov(as
, ir
); break;
1839 case IR_SUBOV
: asm_arithov(as
, ir
); break;
1840 case IR_MULOV
: asm_mulov(as
, ir
); break;
1842 /* Memory references. */
1843 case IR_AREF
: asm_aref(as
, ir
); break;
1844 case IR_HREF
: asm_href(as
, ir
); break;
1845 case IR_HREFK
: asm_hrefk(as
, ir
); break;
1846 case IR_NEWREF
: asm_newref(as
, ir
); break;
1847 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
1848 case IR_FREF
: asm_fref(as
, ir
); break;
1849 case IR_STRREF
: asm_strref(as
, ir
); break;
1851 /* Loads and stores. */
1852 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
1853 asm_ahuvload(as
, ir
);
1855 case IR_FLOAD
: asm_fload(as
, ir
); break;
1856 case IR_XLOAD
: asm_xload(as
, ir
); break;
1857 case IR_SLOAD
: asm_sload(as
, ir
); break;
1859 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
1860 case IR_FSTORE
: asm_fstore(as
, ir
); break;
1861 case IR_XSTORE
: asm_xstore(as
, ir
, 0); break;
1864 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
1865 case IR_TNEW
: asm_tnew(as
, ir
); break;
1866 case IR_TDUP
: asm_tdup(as
, ir
); break;
1867 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
1869 /* Write barriers. */
1870 case IR_TBAR
: asm_tbar(as
, ir
); break;
1871 case IR_OBAR
: asm_obar(as
, ir
); break;
1873 /* Type conversions. */
1874 case IR_CONV
: asm_conv(as
, ir
); break;
1875 case IR_TOBIT
: asm_tobit(as
, ir
); break;
1876 case IR_TOSTR
: asm_tostr(as
, ir
); break;
1877 case IR_STRTO
: asm_strto(as
, ir
); break;
1880 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
1881 case IR_CALLXS
: asm_callx(as
, ir
); break;
1882 case IR_CARG
: break;
1885 setintV(&as
->J
->errinfo
, ir
->o
);
1886 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
1891 /* -- Trace setup --------------------------------------------------------- */
1893 /* Ensure there are enough stack slots for call arguments. */
1894 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
1896 IRRef args
[CCI_NARGS_MAX
];
1897 uint32_t i
, nargs
= (int)CCI_NARGS(ci
);
1898 int nslots
= 4, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
1899 asm_collectargs(as
, ir
, ci
, args
);
1900 for (i
= 0; i
< nargs
; i
++) {
1901 if (args
[i
] && irt_isfp(IR(args
[i
])->t
) &&
1902 nfpr
> 0 && !(ci
->flags
& CCI_VARARG
)) {
1904 ngpr
-= irt_isnum(IR(args
[i
])->t
) ? 2 : 1;
1905 } else if (args
[i
] && irt_isnum(IR(args
[i
])->t
)) {
1908 if (ngpr
> 0) ngpr
-= 2; else nslots
= (nslots
+3) & ~1;
1911 if (ngpr
> 0) ngpr
--; else nslots
++;
1914 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
1915 as
->evenspill
= nslots
;
1916 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
1919 static void asm_setup_target(ASMState
*as
)
1921 asm_sparejump_setup(as
);
1922 asm_exitstub_setup(as
);
1925 /* -- Trace patching ------------------------------------------------------ */
1927 /* Patch exit jumps of existing machine code to a new target. */
1928 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
1930 MCode
*p
= T
->mcode
;
1931 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
1932 MCode
*px
= exitstub_trace_addr(T
, exitno
);
1933 MCode
*cstart
= NULL
, *cstop
= NULL
;
1934 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
1935 MCode exitload
= MIPSI_LI
| MIPSF_T(RID_TMP
) | exitno
;
1936 MCode tjump
= MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
1937 for (p
++; p
< pe
; p
++) {
1938 if (*p
== exitload
) { /* Look for load of exit number. */
1939 if (((p
[-1] ^ (px
-p
)) & 0xffffu
) == 0) { /* Look for exitstub branch. */
1940 ptrdiff_t delta
= target
- p
;
1941 if (((delta
+ 0x8000) >> 16) == 0) { /* Patch in-range branch. */
1943 p
[-1] = (p
[-1] & 0xffff0000u
) | (delta
& 0xffffu
);
1944 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
1946 if (!cstart
) cstart
= p
-1;
1947 } else { /* Branch out of range. Use spare jump slot in mcarea. */
1949 for (i
= 2; i
< 2+MIPS_SPAREJUMP
*2; i
+= 2) {
1950 if (mcarea
[i
] == tjump
) {
1951 delta
= mcarea
+i
- p
;
1953 } else if (mcarea
[i
] == MIPSI_NOP
) {
1956 delta
= mcarea
+i
- p
;
1960 /* Ignore jump slot overflow. Child trace is simply not attached. */
1962 } else if (p
+1 == pe
) {
1963 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
1964 lua_assert(p
[1] == MIPSI_NOP
);
1966 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
1968 if (!cstart
) cstart
= p
+1;
1972 if (cstart
) lj_mcode_sync(cstart
, cstop
);
1973 lj_mcode_patch(J
, mcarea
, 1);