2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate a register or RID_ZERO. */
22 static Reg
ra_alloc1z(ASMState
*as
, IRRef ref
, RegSet allow
)
26 if (!(allow
& RSET_FPR
) && irref_isk(ref
) && get_kval(as
, ref
) == 0)
28 r
= ra_allocref(as
, ref
, allow
);
35 /* Allocate two source registers for three-operand instructions. */
36 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
38 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
39 Reg left
= irl
->r
, right
= irr
->r
;
40 if (ra_hasreg(left
)) {
43 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
46 } else if (ra_hasreg(right
)) {
48 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
49 } else if (ra_hashint(right
)) {
50 right
= ra_alloc1z(as
, ir
->op2
, allow
);
51 left
= ra_alloc1z(as
, ir
->op1
, rset_exclude(allow
, right
));
53 left
= ra_alloc1z(as
, ir
->op1
, allow
);
54 right
= ra_alloc1z(as
, ir
->op2
, rset_exclude(allow
, left
));
56 return left
| (right
<< 8);
59 /* -- Guard handling ------------------------------------------------------ */
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
64 /* Setup spare long-range jump slots per mcarea. */
65 static void asm_sparejump_setup(ASMState
*as
)
67 MCode
*mxp
= as
->mctop
;
68 if ((char *)mxp
== (char *)as
->J
->mcarea
+ as
->J
->szmcarea
) {
69 mxp
-= MIPS_SPAREJUMP
*2;
70 lj_assertA(MIPSI_NOP
== 0, "bad NOP");
71 memset(mxp
, 0, MIPS_SPAREJUMP
*2*sizeof(MCode
));
76 static MCode
*asm_sparejump_use(MCode
*mcarea
, MCode tjump
)
78 MCode
*mxp
= (MCode
*)((char *)mcarea
+ ((MCLink
*)mcarea
)->size
);
79 int slot
= MIPS_SPAREJUMP
;
84 } else if (*mxp
== MIPSI_NOP
) {
92 /* Setup exit stub after the end of each trace. */
93 static void asm_exitstub_setup(ASMState
*as
)
95 MCode
*mxp
= as
->mctop
;
96 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
97 *--mxp
= MIPSI_LI
|MIPSF_T(RID_TMP
)|as
->T
->traceno
;
98 *--mxp
= MIPSI_J
|((((uintptr_t)(void *)lj_vm_exit_handler
)>>2)&0x03ffffffu
);
99 lj_assertA(((uintptr_t)mxp
^ (uintptr_t)(void *)lj_vm_exit_handler
)>>28 == 0,
100 "branch target out of range");
101 *--mxp
= MIPSI_SW
|MIPSF_T(RID_TMP
)|MIPSF_S(RID_SP
)|0;
105 /* Keep this in-sync with exitstub_trace_addr(). */
106 #define asm_exitstub_addr(as) ((as)->mctop)
108 /* Emit conditional branch to exit for guard. */
109 static void asm_guard(ASMState
*as
, MIPSIns mi
, Reg rs
, Reg rt
)
111 MCode
*target
= asm_exitstub_addr(as
);
113 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
117 #if !LJ_TARGET_MIPSR6
118 mi
= mi
^ ((mi
>>28) == 1 ? 0x04000000u
: 0x00010000u
); /* Invert cond. */
120 mi
= mi
^ ((mi
>>28) == 1 ? 0x04000000u
:
121 (mi
>>28) == 4 ? 0x00800000u
: 0x00010000u
); /* Invert cond. */
123 target
= p
; /* Patch target later in asm_loop_fixup. */
125 emit_ti(as
, MIPSI_LI
, RID_TMP
, as
->snapno
);
126 emit_branch(as
, mi
, rs
, rt
, target
);
129 /* -- Operand fusion ------------------------------------------------------ */
131 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
132 #define CONFLICT_SEARCH_LIM 31
134 /* Check if there's no conflicting instruction between curins and ref. */
135 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
138 IRRef i
= as
->curins
;
139 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
140 return 0; /* Give up, ref is too far away. */
142 if (ir
[i
].o
== conflict
)
143 return 0; /* Conflict found. */
144 return 1; /* Ok, no conflict. */
147 /* Fuse the array base of colocated arrays. */
148 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
151 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
152 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
153 return (int32_t)sizeof(GCtab
);
157 /* Fuse array/hash/upvalue reference into register+offset operand. */
158 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
161 if (ra_noreg(ir
->r
)) {
162 if (ir
->o
== IR_AREF
) {
163 if (mayfuse(as
, ref
)) {
164 if (irref_isk(ir
->op2
)) {
165 IRRef tab
= IR(ir
->op1
)->op1
;
166 int32_t ofs
= asm_fuseabase(as
, tab
);
167 IRRef refa
= ofs
? tab
: ir
->op1
;
168 ofs
+= 8*IR(ir
->op2
)->i
;
171 return ra_alloc1(as
, refa
, allow
);
175 } else if (ir
->o
== IR_HREFK
) {
176 if (mayfuse(as
, ref
)) {
177 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
180 return ra_alloc1(as
, ir
->op1
, allow
);
183 } else if (ir
->o
== IR_UREFC
) {
184 if (irref_isk(ir
->op1
)) {
185 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
186 intptr_t ofs
= (intptr_t)&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
;
187 intptr_t jgl
= (intptr_t)J2G(as
->J
);
188 if ((uintptr_t)(ofs
-jgl
) < 65536) {
189 *ofsp
= ofs
-jgl
-32768;
192 *ofsp
= (int16_t)ofs
;
193 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
196 } else if (ir
->o
== IR_TMPREF
) {
197 *ofsp
= (int32_t)(offsetof(global_State
, tmptv
)-32768);
202 return ra_alloc1(as
, ref
, allow
);
205 /* Fuse XLOAD/XSTORE reference into load/store operand. */
206 static void asm_fusexref(ASMState
*as
, MIPSIns mi
, Reg rt
, IRRef ref
,
207 RegSet allow
, int32_t ofs
)
211 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
212 if (ir
->o
== IR_ADD
) {
214 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ get_kval(as
, ir
->op2
),
219 } else if (ir
->o
== IR_STRREF
) {
220 intptr_t ofs2
= 65536;
221 lj_assertA(ofs
== 0, "bad usage");
222 ofs
= (int32_t)sizeof(GCstr
);
223 if (irref_isk(ir
->op2
)) {
224 ofs2
= ofs
+ get_kval(as
, ir
->op2
);
226 } else if (irref_isk(ir
->op1
)) {
227 ofs2
= ofs
+ get_kval(as
, ir
->op1
);
230 if (!checki16(ofs2
)) {
231 /* NYI: Fuse ADD with constant. */
232 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
233 right
= (left
>> 8); left
&= 255;
234 emit_hsi(as
, mi
, rt
, RID_TMP
, ofs
);
235 emit_dst(as
, MIPSI_AADDU
, RID_TMP
, left
, right
);
241 base
= ra_alloc1(as
, ref
, allow
);
242 emit_hsi(as
, mi
, rt
, base
, ofs
);
245 /* -- Calls --------------------------------------------------------------- */
247 /* Generate a call to a C function. */
248 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
250 uint32_t n
, nargs
= CCI_XNARGS(ci
);
251 int32_t ofs
= LJ_32
? 16 : 0;
253 Reg gpr
= REGARG_FIRSTGPR
;
255 Reg gpr
, fpr
= REGARG_FIRSTFPR
;
257 if ((void *)ci
->func
)
258 emit_call(as
, (void *)ci
->func
, 1);
260 for (gpr
= REGARG_FIRSTGPR
; gpr
<= REGARG_LASTGPR
; gpr
++)
261 as
->cost
[gpr
] = REGCOST(~0u, ASMREF_L
);
262 gpr
= REGARG_FIRSTGPR
;
264 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
269 if (irt_isfp(ir
->t
) && fpr
<= REGARG_LASTFPR
&&
270 !(ci
->flags
& CCI_VARARG
)) {
271 lj_assertA(rset_test(as
->freeset
, fpr
),
272 "reg %d not free", fpr
); /* Already evicted. */
273 ra_leftov(as
, fpr
, ref
);
274 fpr
+= LJ_32
? 2 : 1;
275 gpr
+= (LJ_32
&& irt_isnum(ir
->t
)) ? 2 : 1;
279 #if LJ_32 && !LJ_SOFTFP
280 fpr
= REGARG_LASTFPR
+1;
282 if (LJ_32
&& irt_isnum(ir
->t
)) gpr
= (gpr
+1) & ~1;
283 if (gpr
<= REGARG_LASTGPR
) {
284 lj_assertA(rset_test(as
->freeset
, gpr
),
285 "reg %d not free", gpr
); /* Already evicted. */
287 if (irt_isfp(ir
->t
)) {
288 RegSet of
= as
->freeset
;
290 /* Workaround to protect argument GPRs from being used for remat. */
291 as
->freeset
&= ~RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1);
292 r
= ra_alloc1(as
, ref
, RSET_FPR
);
293 as
->freeset
|= (of
& RSET_RANGE(REGARG_FIRSTGPR
, REGARG_LASTGPR
+1));
294 if (irt_isnum(ir
->t
)) {
296 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?0:1), r
+1);
297 emit_tg(as
, MIPSI_MFC1
, gpr
+(LJ_BE
?1:0), r
);
298 lj_assertA(rset_test(as
->freeset
, gpr
+1),
299 "reg %d not free", gpr
+1); /* Already evicted. */
302 emit_tg(as
, MIPSI_DMFC1
, gpr
, r
);
305 } else if (irt_isfloat(ir
->t
)) {
306 emit_tg(as
, MIPSI_MFC1
, gpr
, r
);
315 ra_leftov(as
, gpr
, ref
);
317 #if LJ_64 && !LJ_SOFTFP
322 Reg r
= ra_alloc1z(as
, ref
, !LJ_SOFTFP
&& irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
324 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
325 emit_spstore(as
, ir
, r
, ofs
);
326 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
328 emit_spstore(as
, ir
, r
, ofs
+ ((LJ_BE
&& !irt_isfp(ir
->t
) && !irt_is64(ir
->t
)) ? 4 : 0));
335 fpr
= REGARG_LASTFPR
+1;
337 if (gpr
<= REGARG_LASTGPR
) {
339 #if LJ_64 && !LJ_SOFTFP
343 ofs
+= LJ_32
? 4 : 8;
350 /* Setup result reg/sp for call. Evict scratch regs. */
351 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
353 RegSet drop
= RSET_SCRATCH
;
354 int hiop
= ((ir
+1)->o
== IR_HIOP
&& !irt_isnil((ir
+1)->t
));
356 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
359 if (ra_hasreg(ir
->r
))
360 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
361 if (hiop
&& ra_hasreg((ir
+1)->r
))
362 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
363 ra_evictset(as
, drop
); /* Evictions must be performed first. */
365 lj_assertA(!irt_ispri(ir
->t
), "PRI dest");
366 if (!LJ_SOFTFP
&& irt_isfp(ir
->t
)) {
367 if ((ci
->flags
& CCI_CASTU64
)) {
368 int32_t ofs
= sps_scale(ir
->s
);
370 if (ra_hasreg(dest
)) {
372 ra_modified(as
, dest
);
374 emit_tg(as
, MIPSI_MTC1
, RID_RETHI
, dest
+1);
375 emit_tg(as
, MIPSI_MTC1
, RID_RETLO
, dest
);
377 emit_tg(as
, MIPSI_DMTC1
, RID_RET
, dest
);
382 emit_tsi(as
, MIPSI_SW
, RID_RETLO
, RID_SP
, ofs
+(LJ_BE
?4:0));
383 emit_tsi(as
, MIPSI_SW
, RID_RETHI
, RID_SP
, ofs
+(LJ_BE
?0:4));
385 emit_tsi(as
, MIPSI_SD
, RID_RET
, RID_SP
, ofs
);
389 ra_destreg(as
, ir
, RID_FPRET
);
394 ra_destreg(as
, ir
, RID_RET
);
399 static void asm_callx(ASMState
*as
, IRIns
*ir
)
401 IRRef args
[CCI_NARGS_MAX
*2];
405 ci
.flags
= asm_callx_flags(as
, ir
);
406 asm_collectargs(as
, ir
, &ci
, args
);
407 asm_setupresult(as
, ir
, &ci
);
408 func
= ir
->op2
; irf
= IR(func
);
409 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
410 if (irref_isk(func
)) { /* Call to constant address. */
411 ci
.func
= (ASMFunction
)(void *)get_kval(as
, func
);
412 } else { /* Need specific register for indirect calls. */
413 Reg r
= ra_alloc1(as
, func
, RID2RSET(RID_CFUNCADDR
));
415 if (r
== RID_CFUNCADDR
)
418 *--p
= MIPSI_MOVE
| MIPSF_D(RID_CFUNCADDR
) | MIPSF_S(r
);
419 *--p
= MIPSI_JALR
| MIPSF_S(r
);
421 ci
.func
= (ASMFunction
)(void *)0;
423 asm_gencall(as
, &ci
, args
);
427 static void asm_callround(ASMState
*as
, IRIns
*ir
, IRCallID id
)
429 /* The modified regs must match with the *.dasc implementation. */
430 RegSet drop
= RID2RSET(RID_R1
)|RID2RSET(RID_R12
)|RID2RSET(RID_FPRET
)|
431 RID2RSET(RID_F2
)|RID2RSET(RID_F4
)|RID2RSET(REGARG_FIRSTFPR
)
436 if (ra_hasreg(ir
->r
)) rset_clear(drop
, ir
->r
);
437 ra_evictset(as
, drop
);
438 ra_destreg(as
, ir
, RID_FPRET
);
439 emit_call(as
, (void *)lj_ir_callinfo
[id
].func
, 0);
440 ra_leftov(as
, REGARG_FIRSTFPR
, ir
->op1
);
444 /* -- Returns ------------------------------------------------------------- */
446 /* Return to lower frame. Guard that it goes to the right spot. */
447 static void asm_retf(ASMState
*as
, IRIns
*ir
)
449 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
450 void *pc
= ir_kptr(IR(ir
->op2
));
451 int32_t delta
= 1+LJ_FR2
+bc_a(*((const BCIns
*)pc
- 1));
452 as
->topslot
-= (BCReg
)delta
;
453 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
454 irt_setmark(IR(REF_BASE
)->t
); /* Children must not coalesce with BASE reg. */
455 emit_setgl(as
, base
, jit_base
);
456 emit_addptr(as
, base
, -8*delta
);
457 asm_guard(as
, MIPSI_BNE
, RID_TMP
,
458 ra_allock(as
, igcptr(pc
), rset_exclude(RSET_GPR
, base
)));
459 emit_tsi(as
, MIPSI_AL
, RID_TMP
, base
, (LJ_BE
|| LJ_FR2
) ? -8 : -4);
462 /* -- Buffer operations --------------------------------------------------- */
465 static void asm_bufhdr_write(ASMState
*as
, Reg sb
)
467 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, sb
));
469 irgc
.ot
= IRT(0, IRT_PGC
); /* GC type. */
470 emit_storeofs(as
, &irgc
, RID_TMP
, sb
, offsetof(SBuf
, L
));
471 if ((as
->flags
& JIT_F_MIPSXXR2
)) {
472 emit_tsml(as
, LJ_64
? MIPSI_DINS
: MIPSI_INS
, RID_TMP
, tmp
,
473 lj_fls(SBUF_MASK_FLAG
), 0);
475 emit_dst(as
, MIPSI_OR
, RID_TMP
, RID_TMP
, tmp
);
476 emit_tsi(as
, MIPSI_ANDI
, tmp
, tmp
, SBUF_MASK_FLAG
);
478 emit_getgl(as
, RID_TMP
, cur_L
);
479 emit_loadofs(as
, &irgc
, tmp
, sb
, offsetof(SBuf
, L
));
483 /* -- Type conversions ---------------------------------------------------- */
486 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
488 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
489 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
490 #if !LJ_TARGET_MIPSR6
491 asm_guard(as
, MIPSI_BC1F
, 0, 0);
492 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmp
, left
);
494 asm_guard(as
, MIPSI_BC1EQZ
, 0, (tmp
&31));
495 emit_fgh(as
, MIPSI_CMP_EQ_D
, tmp
, tmp
, left
);
497 emit_fg(as
, MIPSI_CVT_D_W
, tmp
, tmp
);
498 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
499 emit_fg(as
, MIPSI_CVT_W_D
, tmp
, left
);
502 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
504 RegSet allow
= RSET_FPR
;
505 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
506 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
507 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
508 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
509 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
510 emit_fgh(as
, MIPSI_ADD_D
, tmp
, left
, right
);
512 #elif LJ_64 /* && LJ_SOFTFP */
513 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg r
)
515 /* The modified regs must match with the *.dasc implementation. */
516 RegSet drop
= RID2RSET(REGARG_FIRSTGPR
)|RID2RSET(RID_RET
)|RID2RSET(RID_RET
+1)|
517 RID2RSET(RID_R1
)|RID2RSET(RID_R12
);
518 if (ra_hasreg(ir
->r
)) rset_clear(drop
, ir
->r
);
519 ra_evictset(as
, drop
);
520 /* Return values are in RID_RET (converted value) and RID_RET+1 (status). */
521 ra_destreg(as
, ir
, RID_RET
);
522 asm_guard(as
, MIPSI_BNE
, RID_RET
+1, RID_ZERO
);
523 emit_call(as
, (void *)lj_ir_callinfo
[IRCALL_lj_vm_tointg
].func
, 0);
525 ra_leftov(as
, REGARG_FIRSTGPR
, ir
->op1
);
526 else if (r
!= REGARG_FIRSTGPR
)
527 emit_move(as
, REGARG_FIRSTGPR
, r
);
530 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
532 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
533 emit_dta(as
, MIPSI_SLL
, dest
, dest
, 0);
534 asm_callid(as
, ir
, IRCALL_lj_vm_tobit
);
538 static void asm_conv(ASMState
*as
, IRIns
*ir
)
540 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
542 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
545 int st64
= (st
== IRT_I64
|| st
== IRT_U64
|| st
== IRT_P64
);
547 IRRef lref
= ir
->op1
;
549 /* 64 bit integer conversions are handled by SPLIT. */
550 lj_assertA(!(irt_isint64(ir
->t
) || (st
== IRT_I64
|| st
== IRT_U64
)),
551 "IR %04d has unsplit 64 bit type",
552 (int)(ir
- as
->ir
) - REF_BIAS
);
555 /* FP conversions are handled by SPLIT. */
556 lj_assertA(!irt_isfp(ir
->t
) && !(st
== IRT_NUM
|| st
== IRT_FLOAT
),
557 "IR %04d has FP type",
558 (int)(ir
- as
->ir
) - REF_BIAS
);
559 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
561 lj_assertA(irt_type(ir
->t
) != st
, "inconsistent types for CONV");
563 if (irt_isfp(ir
->t
)) {
564 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
565 if (stfp
) { /* FP to FP conversion. */
566 emit_fg(as
, st
== IRT_NUM
? MIPSI_CVT_S_D
: MIPSI_CVT_D_S
,
567 dest
, ra_alloc1(as
, lref
, RSET_FPR
));
568 } else if (st
== IRT_U32
) { /* U32 to FP conversion. */
569 /* y = (x ^ 0x8000000) + 2147483648.0 */
570 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
571 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
572 if (irt_isfloat(ir
->t
))
573 emit_fg(as
, MIPSI_CVT_S_D
, dest
, dest
);
574 /* Must perform arithmetic with doubles to keep the precision. */
575 emit_fgh(as
, MIPSI_ADD_D
, dest
, dest
, tmp
);
576 emit_fg(as
, MIPSI_CVT_D_W
, dest
, dest
);
577 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
578 (void *)&as
->J
->k64
[LJ_K64_2P31
], RSET_GPR
);
579 emit_tg(as
, MIPSI_MTC1
, RID_TMP
, dest
);
580 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, left
);
581 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
583 } else if(st
== IRT_U64
) { /* U64 to FP conversion. */
584 /* if (x >= 1u<<63) y = (double)(int64_t)(x&(1u<<63)-1) + pow(2.0, 63) */
585 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
586 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
587 MCLabel l_end
= emit_label(as
);
588 if (irt_isfloat(ir
->t
)) {
589 emit_fgh(as
, MIPSI_ADD_S
, dest
, dest
, tmp
);
590 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31), (void *)&as
->J
->k32
[LJ_K32_2P63
],
591 rset_exclude(RSET_GPR
, left
));
592 emit_fg(as
, MIPSI_CVT_S_L
, dest
, dest
);
594 emit_fgh(as
, MIPSI_ADD_D
, dest
, dest
, tmp
);
595 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31), (void *)&as
->J
->k64
[LJ_K64_2P63
],
596 rset_exclude(RSET_GPR
, left
));
597 emit_fg(as
, MIPSI_CVT_D_L
, dest
, dest
);
599 emit_branch(as
, MIPSI_BGEZ
, left
, RID_ZERO
, l_end
);
600 emit_tg(as
, MIPSI_DMTC1
, RID_TMP
, dest
);
601 emit_tsml(as
, MIPSI_DEXTM
, RID_TMP
, left
, 30, 0);
603 } else { /* Integer to FP conversion. */
604 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
606 emit_fg(as
, irt_isfloat(ir
->t
) ? MIPSI_CVT_S_W
: MIPSI_CVT_D_W
,
608 emit_tg(as
, MIPSI_MTC1
, left
, dest
);
610 MIPSIns mi
= irt_isfloat(ir
->t
) ?
611 (st64
? MIPSI_CVT_S_L
: MIPSI_CVT_S_W
) :
612 (st64
? MIPSI_CVT_D_L
: MIPSI_CVT_D_W
);
613 emit_fg(as
, mi
, dest
, dest
);
614 emit_tg(as
, st64
? MIPSI_DMTC1
: MIPSI_MTC1
, left
, dest
);
617 } else if (stfp
) { /* FP to integer conversion. */
618 if (irt_isguard(ir
->t
)) {
619 /* Checked conversions are only supported from number to int. */
620 lj_assertA(irt_isint(ir
->t
) && st
== IRT_NUM
,
621 "bad type for checked CONV");
622 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
624 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
625 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
626 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
627 if (irt_isu32(ir
->t
)) { /* FP to U32 conversion. */
628 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
629 emit_dst(as
, MIPSI_XOR
, dest
, dest
, RID_TMP
);
630 emit_ti(as
, MIPSI_LUI
, RID_TMP
, 0x8000);
631 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
632 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_FLOOR_W_S
: MIPSI_FLOOR_W_D
,
634 emit_fgh(as
, st
== IRT_FLOAT
? MIPSI_SUB_S
: MIPSI_SUB_D
,
637 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
638 (void *)&as
->J
->k32
[LJ_K32_2P31
], RSET_GPR
);
640 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
641 (void *)&as
->J
->k64
[LJ_K64_2P31
], RSET_GPR
);
643 } else if (irt_isu64(ir
->t
)) { /* FP to U64 conversion. */
645 emit_tg(as
, MIPSI_DMFC1
, dest
, tmp
);
646 l_end
= emit_label(as
);
647 /* For inputs >= 2^63 add -2^64 and convert again. */
649 emit_fg(as
, MIPSI_TRUNC_L_D
, tmp
, tmp
);
650 emit_fgh(as
, MIPSI_ADD_D
, tmp
, left
, tmp
);
651 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
652 (void *)&as
->J
->k64
[LJ_K64_M2P64
],
653 rset_exclude(RSET_GPR
, dest
));
654 emit_fg(as
, MIPSI_TRUNC_L_D
, tmp
, left
); /* Delay slot. */
655 #if !LJ_TARGET_MIPSR6
656 emit_branch(as
, MIPSI_BC1T
, 0, 0, l_end
);
657 emit_fgh(as
, MIPSI_C_OLT_D
, 0, left
, tmp
);
659 emit_branch(as
, MIPSI_BC1NEZ
, 0, (tmp
&31), l_end
);
660 emit_fgh(as
, MIPSI_CMP_LT_D
, tmp
, left
, tmp
);
662 emit_lsptr(as
, MIPSI_LDC1
, (tmp
& 31),
663 (void *)&as
->J
->k64
[LJ_K64_2P63
],
664 rset_exclude(RSET_GPR
, dest
));
666 emit_fg(as
, MIPSI_TRUNC_L_S
, tmp
, tmp
);
667 emit_fgh(as
, MIPSI_ADD_S
, tmp
, left
, tmp
);
668 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
669 (void *)&as
->J
->k32
[LJ_K32_M2P64
],
670 rset_exclude(RSET_GPR
, dest
));
671 emit_fg(as
, MIPSI_TRUNC_L_S
, tmp
, left
); /* Delay slot. */
672 #if !LJ_TARGET_MIPSR6
673 emit_branch(as
, MIPSI_BC1T
, 0, 0, l_end
);
674 emit_fgh(as
, MIPSI_C_OLT_S
, 0, left
, tmp
);
676 emit_branch(as
, MIPSI_BC1NEZ
, 0, (tmp
&31), l_end
);
677 emit_fgh(as
, MIPSI_CMP_LT_S
, tmp
, left
, tmp
);
679 emit_lsptr(as
, MIPSI_LWC1
, (tmp
& 31),
680 (void *)&as
->J
->k32
[LJ_K32_2P63
],
681 rset_exclude(RSET_GPR
, dest
));
686 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
687 emit_fg(as
, st
== IRT_FLOAT
? MIPSI_TRUNC_W_S
: MIPSI_TRUNC_W_D
,
690 MIPSIns mi
= irt_is64(ir
->t
) ?
691 (st
== IRT_NUM
? MIPSI_TRUNC_L_D
: MIPSI_TRUNC_L_S
) :
692 (st
== IRT_NUM
? MIPSI_TRUNC_W_D
: MIPSI_TRUNC_W_S
);
693 emit_tg(as
, irt_is64(ir
->t
) ? MIPSI_DMFC1
: MIPSI_MFC1
, dest
, tmp
);
694 emit_fg(as
, mi
, tmp
, left
);
700 if (irt_isfp(ir
->t
)) {
701 #if LJ_64 && LJ_HASFFI
702 if (stfp
) { /* FP to FP conversion. */
703 asm_callid(as
, ir
, irt_isnum(ir
->t
) ? IRCALL_softfp_f2d
:
705 } else { /* Integer to FP conversion. */
706 IRCallID cid
= ((IRT_IS64
>> st
) & 1) ?
708 (st
== IRT_I64
? IRCALL_fp64_l2d
: IRCALL_fp64_ul2d
) :
709 (st
== IRT_I64
? IRCALL_fp64_l2f
: IRCALL_fp64_ul2f
)) :
711 (st
== IRT_INT
? IRCALL_softfp_i2d
: IRCALL_softfp_ui2d
) :
712 (st
== IRT_INT
? IRCALL_softfp_i2f
: IRCALL_softfp_ui2f
));
713 asm_callid(as
, ir
, cid
);
716 asm_callid(as
, ir
, IRCALL_softfp_i2d
);
718 } else if (stfp
) { /* FP to integer conversion. */
719 if (irt_isguard(ir
->t
)) {
720 /* Checked conversions are only supported from number to int. */
721 lj_assertA(irt_isint(ir
->t
) && st
== IRT_NUM
,
722 "bad type for checked CONV");
723 asm_tointg(as
, ir
, RID_NONE
);
725 IRCallID cid
= irt_is64(ir
->t
) ?
727 (irt_isi64(ir
->t
) ? IRCALL_fp64_d2l
: IRCALL_fp64_d2ul
) :
728 (irt_isi64(ir
->t
) ? IRCALL_fp64_f2l
: IRCALL_fp64_f2ul
)) :
730 (irt_isint(ir
->t
) ? IRCALL_softfp_d2i
: IRCALL_softfp_d2ui
) :
731 (irt_isint(ir
->t
) ? IRCALL_softfp_f2i
: IRCALL_softfp_f2ui
));
732 asm_callid(as
, ir
, cid
);
738 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
739 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
740 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
741 lj_assertA(irt_isint(ir
->t
) || irt_isu32(ir
->t
), "bad type for CONV EXT");
742 if ((ir
->op2
& IRCONV_SEXT
)) {
743 if (LJ_64
|| (as
->flags
& JIT_F_MIPSXXR2
)) {
744 emit_dst(as
, st
== IRT_I8
? MIPSI_SEB
: MIPSI_SEH
, dest
, 0, left
);
746 uint32_t shift
= st
== IRT_I8
? 24 : 16;
747 emit_dta(as
, MIPSI_SRA
, dest
, dest
, shift
);
748 emit_dta(as
, MIPSI_SLL
, dest
, left
, shift
);
751 emit_tsi(as
, MIPSI_ANDI
, dest
, left
,
752 (int32_t)(st
== IRT_U8
? 0xff : 0xffff));
754 } else { /* 32/64 bit integer conversions. */
756 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
757 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
759 if (irt_is64(ir
->t
)) {
761 /* 64/64 bit no-op (cast)*/
762 ra_leftov(as
, dest
, lref
);
764 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
765 if ((ir
->op2
& IRCONV_SEXT
)) { /* 32 to 64 bit sign extension. */
766 emit_dta(as
, MIPSI_SLL
, dest
, left
, 0);
767 } else { /* 32 to 64 bit zero extension. */
768 emit_tsml(as
, MIPSI_DEXT
, dest
, left
, 31, 0);
772 if (st64
&& !(ir
->op2
& IRCONV_NONE
)) {
773 /* This is either a 32 bit reg/reg mov which zeroes the hiword
774 ** or a load of the loword from a 64 bit address.
776 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
777 emit_tsml(as
, MIPSI_DEXT
, dest
, left
, 31, 0);
778 } else { /* 32/32 bit no-op (cast). */
779 /* Do nothing, but may need to move regs. */
780 ra_leftov(as
, dest
, lref
);
788 static void asm_strto(ASMState
*as
, IRIns
*ir
)
790 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
794 ra_evictset(as
, RSET_SCRATCH
);
796 if (ra_hasspill(ir
->s
) && ra_hasspill((ir
+1)->s
) &&
797 (ir
->s
& 1) == LJ_BE
&& (ir
->s
^ 1) == (ir
+1)->s
) {
799 for (i
= 0; i
< 2; i
++) {
804 emit_spload(as
, ir
+i
, r
, sps_scale((ir
+i
)->s
));
807 ofs
= sps_scale(ir
->s
& ~1);
809 Reg rhi
= ra_dest(as
, ir
+1, RSET_GPR
);
810 Reg rlo
= ra_dest(as
, ir
, rset_exclude(RSET_GPR
, rhi
));
811 emit_tsi(as
, MIPSI_LW
, rhi
, RID_SP
, ofs
+(LJ_BE
?0:4));
812 emit_tsi(as
, MIPSI_LW
, rlo
, RID_SP
, ofs
+(LJ_BE
?4:0));
816 RegSet drop
= RSET_SCRATCH
;
817 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
818 ra_evictset(as
, drop
);
819 ofs
= sps_scale(ir
->s
);
821 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_ZERO
); /* Test return status. */
822 args
[0] = ir
->op1
; /* GCstr *str */
823 args
[1] = ASMREF_TMP1
; /* TValue *n */
824 asm_gencall(as
, ci
, args
);
825 /* Store the result to the spill slot or temp slots. */
826 emit_tsi(as
, MIPSI_AADDIU
, ra_releasetmp(as
, ASMREF_TMP1
),
830 /* -- Memory references --------------------------------------------------- */
833 /* Store tagged value for ref at base+ofs. */
834 static void asm_tvstore64(ASMState
*as
, Reg base
, int32_t ofs
, IRRef ref
)
836 RegSet allow
= rset_exclude(RSET_GPR
, base
);
838 lj_assertA(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
),
839 "store of IR type %d", irt_type(ir
->t
));
840 if (irref_isk(ref
)) {
842 lj_ir_kvalue(as
->J
->L
, &k
, ir
);
843 emit_tsi(as
, MIPSI_SD
, ra_allock(as
, (int64_t)k
.u64
, allow
), base
, ofs
);
845 Reg src
= ra_alloc1(as
, ref
, allow
);
846 Reg type
= ra_allock(as
, (int64_t)irt_toitype(ir
->t
) << 47,
847 rset_exclude(allow
, src
));
848 emit_tsi(as
, MIPSI_SD
, RID_TMP
, base
, ofs
);
849 if (irt_isinteger(ir
->t
)) {
850 emit_dst(as
, MIPSI_DADDU
, RID_TMP
, RID_TMP
, type
);
851 emit_tsml(as
, MIPSI_DEXT
, RID_TMP
, src
, 31, 0);
853 emit_dst(as
, MIPSI_DADDU
, RID_TMP
, src
, type
);
859 /* Get pointer to TValue. */
860 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
, MSize mode
)
862 int32_t tmpofs
= (int32_t)(offsetof(global_State
, tmptv
)-32768);
863 if ((mode
& IRTMPREF_IN1
)) {
865 if (irt_isnum(ir
->t
)) {
866 if ((mode
& IRTMPREF_OUT1
)) {
868 emit_tsi(as
, MIPSI_AADDIU
, dest
, RID_JGL
, tmpofs
);
870 emit_setgl(as
, ra_alloc1(as
, ref
, RSET_GPR
), tmptv
.u64
);
872 lj_assertA(irref_isk(ref
), "unsplit FP op");
874 ra_allock(as
, (int32_t)ir_knum(ir
)->u32
.lo
, RSET_GPR
),
877 ra_allock(as
, (int32_t)ir_knum(ir
)->u32
.hi
, RSET_GPR
),
881 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
882 emit_tsi(as
, MIPSI_AADDIU
, dest
, RID_JGL
, tmpofs
);
883 emit_tsi(as
, MIPSI_SDC1
, (src
& 31), RID_JGL
, tmpofs
);
885 } else if (irref_isk(ref
)) {
886 /* Use the number constant itself as a TValue. */
887 ra_allockreg(as
, igcptr(ir_knum(ir
)), dest
);
890 lj_assertA(0, "unsplit FP op");
892 /* Otherwise force a spill and use the spill slot. */
893 emit_tsi(as
, MIPSI_AADDIU
, dest
, RID_SP
, ra_spill(as
, ir
));
897 /* Otherwise use g->tmptv to hold the TValue. */
900 emit_tsi(as
, MIPSI_ADDIU
, dest
, RID_JGL
, tmpofs
);
901 if (!irt_ispri(ir
->t
)) {
902 Reg src
= ra_alloc1(as
, ref
, RSET_GPR
);
903 emit_setgl(as
, src
, tmptv
.gcr
);
905 if (LJ_SOFTFP
&& (ir
+1)->o
== IR_HIOP
&& !irt_isnil((ir
+1)->t
))
906 type
= ra_alloc1(as
, ref
+1, RSET_GPR
);
908 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), RSET_GPR
);
909 emit_setgl(as
, type
, tmptv
.it
);
911 asm_tvstore64(as
, dest
, 0, ref
);
912 emit_tsi(as
, MIPSI_DADDIU
, dest
, RID_JGL
, tmpofs
);
916 emit_tsi(as
, MIPSI_AADDIU
, dest
, RID_JGL
, tmpofs
);
920 static void asm_aref(ASMState
*as
, IRIns
*ir
)
922 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
924 if (irref_isk(ir
->op2
)) {
925 IRRef tab
= IR(ir
->op1
)->op1
;
926 int32_t ofs
= asm_fuseabase(as
, tab
);
927 IRRef refa
= ofs
? tab
: ir
->op1
;
928 ofs
+= 8*IR(ir
->op2
)->i
;
930 base
= ra_alloc1(as
, refa
, RSET_GPR
);
931 emit_tsi(as
, MIPSI_AADDIU
, dest
, base
, ofs
);
935 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
936 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
937 #if !LJ_TARGET_MIPSR6
938 emit_dst(as
, MIPSI_AADDU
, dest
, RID_TMP
, base
);
939 emit_dta(as
, MIPSI_SLL
, RID_TMP
, idx
, 3);
941 emit_dst(as
, MIPSI_ALSA
| MIPSF_A(3-1), dest
, idx
, base
);
945 /* Inlined hash lookup. Specialized for key type and for const keys.
946 ** The equivalent C code is:
947 ** Node *n = hashkey(t, key);
949 ** if (lj_obj_equal(&n->key, key)) return &n->val;
950 ** } while ((n = nextnode(n)));
953 static void asm_href(ASMState
*as
, IRIns
*ir
, IROp merge
)
955 RegSet allow
= RSET_GPR
;
956 int destused
= ra_used(ir
);
957 Reg dest
= ra_dest(as
, ir
, allow
);
958 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
959 Reg key
= RID_NONE
, type
= RID_NONE
, tmpnum
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
961 Reg cmp64
= RID_NONE
;
963 IRRef refkey
= ir
->op2
;
964 IRIns
*irkey
= IR(refkey
);
965 int isk
= irref_isk(refkey
);
966 IRType1 kt
= irkey
->t
;
968 MCLabel l_end
, l_loop
, l_next
;
970 rset_clear(allow
, tab
);
971 if (!LJ_SOFTFP
&& irt_isnum(kt
)) {
972 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
973 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
975 if (!irt_ispri(kt
)) {
976 key
= ra_alloc1(as
, refkey
, allow
);
977 rset_clear(allow
, key
);
980 if (LJ_SOFTFP
&& irkey
[1].o
== IR_HIOP
) {
981 if (ra_hasreg((irkey
+1)->r
)) {
982 type
= tmpnum
= (irkey
+1)->r
;
983 tmp1
= ra_scratch(as
, allow
);
984 rset_clear(allow
, tmp1
);
985 ra_noweak(as
, tmpnum
);
987 type
= tmpnum
= ra_allocref(as
, refkey
+1, allow
);
989 rset_clear(allow
, tmpnum
);
991 type
= ra_allock(as
, (int32_t)irt_toitype(kt
), allow
);
992 rset_clear(allow
, type
);
996 tmp2
= ra_scratch(as
, allow
);
997 rset_clear(allow
, tmp2
);
999 if (LJ_SOFTFP
|| !irt_isnum(kt
)) {
1000 /* Allocate cmp64 register used for 64-bit comparisons */
1001 if (LJ_SOFTFP
&& irt_isnum(kt
)) {
1003 } else if (!isk
&& irt_isaddr(kt
)) {
1007 if (isk
&& irt_isaddr(kt
)) {
1008 k
= ((int64_t)irt_toitype(kt
) << 47) | irkey
[1].tv
.u64
;
1010 lj_assertA(irt_ispri(kt
) && !irt_isnil(kt
), "bad HREF key type");
1011 k
= ~((int64_t)~irt_toitype(kt
) << 47);
1013 cmp64
= ra_allock(as
, k
, allow
);
1014 rset_clear(allow
, cmp64
);
1019 /* Key not found in chain: jump to exit (if merged) or load niltv. */
1020 l_end
= emit_label(as
);
1023 asm_guard(as
, MIPSI_B
, RID_ZERO
, RID_ZERO
);
1025 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
1026 /* Follow hash chain until the end. */
1027 emit_move(as
, dest
, tmp1
);
1029 emit_tsi(as
, MIPSI_AL
, tmp1
, dest
, (int32_t)offsetof(Node
, next
));
1030 l_next
= emit_label(as
);
1032 /* Type and value comparison. */
1033 if (merge
== IR_EQ
) { /* Must match asm_guard(). */
1034 emit_ti(as
, MIPSI_LI
, RID_TMP
, as
->snapno
);
1035 l_end
= asm_exitstub_addr(as
);
1037 if (!LJ_SOFTFP
&& irt_isnum(kt
)) {
1038 #if !LJ_TARGET_MIPSR6
1039 emit_branch(as
, MIPSI_BC1T
, 0, 0, l_end
);
1040 emit_fgh(as
, MIPSI_C_EQ_D
, 0, tmpnum
, key
);
1042 emit_branch(as
, MIPSI_BC1NEZ
, 0, (tmpnum
&31), l_end
);
1043 emit_fgh(as
, MIPSI_CMP_EQ_D
, tmpnum
, tmpnum
, key
);
1045 *--as
->mcp
= MIPSI_NOP
; /* Avoid NaN comparison overhead. */
1046 emit_branch(as
, MIPSI_BEQ
, tmp1
, RID_ZERO
, l_next
);
1047 emit_tsi(as
, MIPSI_SLTIU
, tmp1
, tmp1
, (int32_t)LJ_TISNUM
);
1049 emit_hsi(as
, MIPSI_LDC1
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
1051 if (irt_ispri(kt
)) {
1052 emit_branch(as
, MIPSI_BEQ
, tmp1
, type
, l_end
);
1054 emit_branch(as
, MIPSI_BEQ
, tmp2
, key
, l_end
);
1055 emit_tsi(as
, MIPSI_LW
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
1056 emit_branch(as
, MIPSI_BNE
, tmp1
, type
, l_next
);
1059 emit_tsi(as
, MIPSI_LW
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
1060 *l_loop
= MIPSI_BNE
| MIPSF_S(tmp1
) | ((as
->mcp
-l_loop
-1) & 0xffffu
);
1062 emit_dta(as
, MIPSI_DSRA32
, tmp1
, tmp1
, 15);
1063 emit_tg(as
, MIPSI_DMTC1
, tmp1
, tmpnum
);
1064 emit_tsi(as
, MIPSI_LD
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.u64
));
1066 emit_branch(as
, MIPSI_BEQ
, tmp1
, cmp64
, l_end
);
1067 emit_tsi(as
, MIPSI_LD
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.u64
));
1069 *l_loop
= MIPSI_BNE
| MIPSF_S(tmp1
) | ((as
->mcp
-l_loop
-1) & 0xffffu
);
1070 if (!isk
&& irt_isaddr(kt
)) {
1071 type
= ra_allock(as
, (int64_t)irt_toitype(kt
) << 47, allow
);
1072 emit_dst(as
, MIPSI_DADDU
, tmp2
, key
, type
);
1073 rset_clear(allow
, type
);
1077 /* Load main position relative to tab->node into dest. */
1078 khash
= isk
? ir_khash(as
, irkey
) : 1;
1080 emit_tsi(as
, MIPSI_AL
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
1084 tmphash
= ra_allock(as
, khash
, allow
);
1085 emit_dst(as
, MIPSI_AADDU
, dest
, dest
, tmp1
);
1086 lj_assertA(sizeof(Node
) == 24, "bad Node size");
1087 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp2
, tmp1
);
1088 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, 3);
1089 emit_dta(as
, MIPSI_SLL
, tmp2
, tmp1
, 5);
1090 emit_dst(as
, MIPSI_AND
, tmp1
, tmp2
, tmphash
);
1091 emit_tsi(as
, MIPSI_AL
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
1092 emit_tsi(as
, MIPSI_LW
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
1094 /* Nothing to do. */
1095 } else if (irt_isstr(kt
)) {
1096 emit_tsi(as
, MIPSI_LW
, tmp1
, key
, (int32_t)offsetof(GCstr
, sid
));
1097 } else { /* Must match with hash*() in lj_tab.c. */
1098 emit_dst(as
, MIPSI_SUBU
, tmp1
, tmp1
, tmp2
);
1099 emit_rotr(as
, tmp2
, tmp2
, dest
, (-HASH_ROT3
)&31);
1100 emit_dst(as
, MIPSI_XOR
, tmp1
, tmp1
, tmp2
);
1101 emit_rotr(as
, tmp1
, tmp1
, dest
, (-HASH_ROT2
-HASH_ROT1
)&31);
1102 emit_dst(as
, MIPSI_SUBU
, tmp2
, tmp2
, dest
);
1104 if (LJ_SOFTFP
? (irkey
[1].o
== IR_HIOP
) : irt_isnum(kt
)) {
1105 emit_dst(as
, MIPSI_XOR
, tmp2
, tmp2
, tmp1
);
1106 if ((as
->flags
& JIT_F_MIPSXXR2
)) {
1107 emit_dta(as
, MIPSI_ROTR
, dest
, tmp1
, (-HASH_ROT1
)&31);
1109 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp1
);
1110 emit_dta(as
, MIPSI_SLL
, tmp1
, tmp1
, HASH_ROT1
);
1111 emit_dta(as
, MIPSI_SRL
, dest
, tmp1
, (-HASH_ROT1
)&31);
1113 emit_dst(as
, MIPSI_ADDU
, tmp1
, tmp1
, tmp1
);
1115 emit_ds(as
, MIPSI_MOVE
, tmp1
, type
);
1116 emit_ds(as
, MIPSI_MOVE
, tmp2
, key
);
1118 emit_tg(as
, MIPSI_MFC1
, tmp2
, key
);
1119 emit_tg(as
, MIPSI_MFC1
, tmp1
, key
+1);
1122 emit_dst(as
, MIPSI_XOR
, tmp2
, key
, tmp1
);
1123 emit_rotr(as
, dest
, tmp1
, tmp2
, (-HASH_ROT1
)&31);
1124 emit_dst(as
, MIPSI_ADDU
, tmp1
, key
, ra_allock(as
, HASH_BIAS
, allow
));
1127 emit_dst(as
, MIPSI_XOR
, tmp2
, tmp2
, tmp1
);
1128 emit_dta(as
, MIPSI_ROTR
, dest
, tmp1
, (-HASH_ROT1
)&31);
1129 if (irt_isnum(kt
)) {
1130 emit_dst(as
, MIPSI_ADDU
, tmp1
, tmp1
, tmp1
);
1131 emit_dta(as
, MIPSI_DSRA32
, tmp1
, LJ_SOFTFP
? key
: tmp1
, 0);
1132 emit_dta(as
, MIPSI_SLL
, tmp2
, LJ_SOFTFP
? key
: tmp1
, 0);
1134 emit_tg(as
, MIPSI_DMFC1
, tmp1
, key
);
1138 emit_dta(as
, MIPSI_DSRA32
, tmp1
, tmp1
, 0);
1139 emit_dta(as
, MIPSI_SLL
, tmp2
, key
, 0);
1140 emit_dst(as
, MIPSI_DADDU
, tmp1
, key
, type
);
1147 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
1149 IRIns
*kslot
= IR(ir
->op2
);
1150 IRIns
*irkey
= IR(kslot
->op1
);
1151 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
1152 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
1153 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
1154 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1155 RegSet allow
= rset_exclude(RSET_GPR
, node
);
1158 Reg key
= RID_NONE
, type
= RID_TMP
;
1161 Reg key
= ra_scratch(as
, allow
);
1164 lj_assertA(ofs
% sizeof(Node
) == 0, "unaligned HREFK slot");
1167 rset_clear(allow
, dest
);
1168 kofs
= (int32_t)offsetof(Node
, key
);
1169 } else if (ra_hasreg(dest
)) {
1170 emit_tsi(as
, MIPSI_AADDIU
, dest
, node
, ofs
);
1173 if (!irt_ispri(irkey
->t
)) {
1174 key
= ra_scratch(as
, allow
);
1175 rset_clear(allow
, key
);
1177 if (irt_isnum(irkey
->t
)) {
1178 lo
= (int32_t)ir_knum(irkey
)->u32
.lo
;
1179 hi
= (int32_t)ir_knum(irkey
)->u32
.hi
;
1182 hi
= irt_toitype(irkey
->t
);
1183 if (!ra_hasreg(key
))
1186 asm_guard(as
, MIPSI_BNE
, key
, lo
? ra_allock(as
, lo
, allow
) : RID_ZERO
);
1188 asm_guard(as
, MIPSI_BNE
, type
, hi
? ra_allock(as
, hi
, allow
) : RID_ZERO
);
1189 if (ra_hasreg(key
)) emit_tsi(as
, MIPSI_LW
, key
, idx
, kofs
+(LJ_BE
?4:0));
1190 emit_tsi(as
, MIPSI_LW
, type
, idx
, kofs
+(LJ_BE
?0:4));
1192 if (irt_ispri(irkey
->t
)) {
1193 lj_assertA(!irt_isnil(irkey
->t
), "bad HREFK key type");
1194 k
= ~((int64_t)~irt_toitype(irkey
->t
) << 47);
1195 } else if (irt_isnum(irkey
->t
)) {
1196 k
= (int64_t)ir_knum(irkey
)->u64
;
1198 k
= ((int64_t)irt_toitype(irkey
->t
) << 47) | (int64_t)ir_kgc(irkey
);
1200 asm_guard(as
, MIPSI_BNE
, key
, ra_allock(as
, k
, allow
));
1201 emit_tsi(as
, MIPSI_LD
, key
, idx
, kofs
);
1204 emit_tsi(as
, MIPSI_AADDU
, dest
, node
, ra_allock(as
, ofs
, allow
));
1207 static void asm_uref(ASMState
*as
, IRIns
*ir
)
1209 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1210 int guarded
= (irt_t(ir
->t
) & (IRT_GUARD
|IRT_TYPE
)) == (IRT_GUARD
|IRT_PGC
);
1211 if (irref_isk(ir
->op1
) && !guarded
) {
1212 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
1213 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
1214 emit_lsptr(as
, MIPSI_AL
, dest
, v
, RSET_GPR
);
1217 asm_guard(as
, ir
->o
== IR_UREFC
? MIPSI_BEQ
: MIPSI_BNE
, RID_TMP
, RID_ZERO
);
1218 if (ir
->o
== IR_UREFC
)
1219 emit_tsi(as
, MIPSI_AADDIU
, dest
, dest
, (int32_t)offsetof(GCupval
, tv
));
1221 emit_tsi(as
, MIPSI_AL
, dest
, dest
, (int32_t)offsetof(GCupval
, v
));
1223 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, dest
, (int32_t)offsetof(GCupval
, closed
));
1224 if (irref_isk(ir
->op1
)) {
1225 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
1226 GCobj
*o
= gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)]);
1227 emit_loada(as
, dest
, o
);
1229 emit_tsi(as
, MIPSI_AL
, dest
, ra_alloc1(as
, ir
->op1
, RSET_GPR
),
1230 (int32_t)offsetof(GCfuncL
, uvptr
) +
1231 (int32_t)sizeof(MRef
) * (int32_t)(ir
->op2
>> 8));
1236 static void asm_fref(ASMState
*as
, IRIns
*ir
)
1238 UNUSED(as
); UNUSED(ir
);
1239 lj_assertA(!ra_used(ir
), "unfused FREF");
1242 static void asm_strref(ASMState
*as
, IRIns
*ir
)
1245 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1246 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
1247 int32_t ofs
= (int32_t)sizeof(GCstr
);
1249 if (irref_isk(ref
)) {
1250 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
1251 } else if (!irref_isk(refk
)) {
1252 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1253 IRIns
*irr
= IR(ir
->op2
);
1254 if (ra_hasreg(irr
->r
)) {
1255 ra_noweak(as
, irr
->r
);
1257 } else if (mayfuse(as
, irr
->op2
) &&
1258 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
1259 checki16(ofs
+ IR(irr
->op2
)->i
)) {
1260 ofs
+= IR(irr
->op2
)->i
;
1261 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
1263 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1265 emit_tsi(as
, MIPSI_ADDIU
, dest
, dest
, ofs
);
1266 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1269 r
= ra_alloc1(as
, ref
, RSET_GPR
);
1272 emit_tsi(as
, MIPSI_ADDIU
, dest
, r
, ofs
);
1274 emit_dst(as
, MIPSI_ADDU
, dest
, r
,
1275 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
1277 RegSet allow
= RSET_GPR
;
1278 Reg dest
= ra_dest(as
, ir
, allow
);
1279 Reg base
= ra_alloc1(as
, ir
->op1
, allow
);
1280 IRIns
*irr
= IR(ir
->op2
);
1281 int32_t ofs
= sizeof(GCstr
);
1282 rset_clear(allow
, base
);
1283 if (irref_isk(ir
->op2
) && checki16(ofs
+ irr
->i
)) {
1284 emit_tsi(as
, MIPSI_DADDIU
, dest
, base
, ofs
+ irr
->i
);
1286 emit_tsi(as
, MIPSI_DADDIU
, dest
, dest
, ofs
);
1287 emit_dst(as
, MIPSI_DADDU
, dest
, base
, ra_alloc1(as
, ir
->op2
, allow
));
1292 /* -- Loads and stores ---------------------------------------------------- */
1294 static MIPSIns
asm_fxloadins(ASMState
*as
, IRIns
*ir
)
1297 switch (irt_type(ir
->t
)) {
1298 case IRT_I8
: return MIPSI_LB
;
1299 case IRT_U8
: return MIPSI_LBU
;
1300 case IRT_I16
: return MIPSI_LH
;
1301 case IRT_U16
: return MIPSI_LHU
;
1303 lj_assertA(!LJ_SOFTFP32
, "unsplit FP op");
1304 if (!LJ_SOFTFP
) return MIPSI_LDC1
;
1306 case IRT_FLOAT
: if (!LJ_SOFTFP
) return MIPSI_LWC1
;
1308 default: return (LJ_64
&& irt_is64(ir
->t
)) ? MIPSI_LD
: MIPSI_LW
;
1312 static MIPSIns
asm_fxstoreins(ASMState
*as
, IRIns
*ir
)
1315 switch (irt_type(ir
->t
)) {
1316 case IRT_I8
: case IRT_U8
: return MIPSI_SB
;
1317 case IRT_I16
: case IRT_U16
: return MIPSI_SH
;
1319 lj_assertA(!LJ_SOFTFP32
, "unsplit FP op");
1320 if (!LJ_SOFTFP
) return MIPSI_SDC1
;
1322 case IRT_FLOAT
: if (!LJ_SOFTFP
) return MIPSI_SWC1
;
1324 default: return (LJ_64
&& irt_is64(ir
->t
)) ? MIPSI_SD
: MIPSI_SW
;
1328 static void asm_fload(ASMState
*as
, IRIns
*ir
)
1330 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1331 MIPSIns mi
= asm_fxloadins(as
, ir
);
1334 if (ir
->op1
== REF_NIL
) { /* FLOAD from GG_State with offset. */
1336 ofs
= (ir
->op2
<< 2) - 32768 - GG_OFS(g
);
1338 idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1339 if (ir
->op2
== IRFL_TAB_ARRAY
) {
1340 ofs
= asm_fuseabase(as
, ir
->op1
);
1341 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
1342 emit_tsi(as
, MIPSI_AADDIU
, dest
, idx
, ofs
);
1346 ofs
= field_ofs
[ir
->op2
];
1347 lj_assertA(!irt_isfp(ir
->t
), "bad FP FLOAD");
1349 emit_tsi(as
, mi
, dest
, idx
, ofs
);
1352 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
1354 if (ir
->r
!= RID_SINK
) {
1355 Reg src
= ra_alloc1z(as
, ir
->op2
, RSET_GPR
);
1356 IRIns
*irf
= IR(ir
->op1
);
1357 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
1358 int32_t ofs
= field_ofs
[irf
->op2
];
1359 MIPSIns mi
= asm_fxstoreins(as
, ir
);
1360 lj_assertA(!irt_isfp(ir
->t
), "bad FP FSTORE");
1361 emit_tsi(as
, mi
, src
, idx
, ofs
);
1365 static void asm_xload(ASMState
*as
, IRIns
*ir
)
1367 Reg dest
= ra_dest(as
, ir
,
1368 (!LJ_SOFTFP
&& irt_isfp(ir
->t
)) ? RSET_FPR
: RSET_GPR
);
1369 lj_assertA(LJ_TARGET_UNALIGNED
|| !(ir
->op2
& IRXLOAD_UNALIGNED
),
1371 asm_fusexref(as
, asm_fxloadins(as
, ir
), dest
, ir
->op1
, RSET_GPR
, 0);
1374 static void asm_xstore_(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
1376 if (ir
->r
!= RID_SINK
) {
1377 Reg src
= ra_alloc1z(as
, ir
->op2
,
1378 (!LJ_SOFTFP
&& irt_isfp(ir
->t
)) ? RSET_FPR
: RSET_GPR
);
1379 asm_fusexref(as
, asm_fxstoreins(as
, ir
), src
, ir
->op1
,
1380 rset_exclude(RSET_GPR
, src
), ofs
);
1384 #define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
1386 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
1388 int hiop
= (LJ_SOFTFP32
&& (ir
+1)->o
== IR_HIOP
);
1389 Reg dest
= RID_NONE
, type
= RID_TMP
, idx
;
1390 RegSet allow
= RSET_GPR
;
1395 if (ra_used(ir
+1)) {
1396 type
= ra_dest(as
, ir
+1, allow
);
1397 rset_clear(allow
, type
);
1401 lj_assertA((LJ_SOFTFP32
? 0 : irt_isnum(ir
->t
)) ||
1402 irt_isint(ir
->t
) || irt_isaddr(ir
->t
),
1403 "bad load type %d", irt_type(ir
->t
));
1404 dest
= ra_dest(as
, ir
, (!LJ_SOFTFP
&& irt_isnum(t
)) ? RSET_FPR
: allow
);
1405 rset_clear(allow
, dest
);
1408 emit_tsml(as
, MIPSI_DEXTM
, dest
, dest
, 14, 0);
1409 else if (irt_isint(t
))
1410 emit_dta(as
, MIPSI_SLL
, dest
, dest
, 0);
1413 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
1414 if (ir
->o
== IR_VLOAD
) ofs
+= 8 * ir
->op2
;
1415 rset_clear(allow
, idx
);
1417 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1418 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, type
, (int32_t)LJ_TISNUM
);
1420 asm_guard(as
, MIPSI_BNE
, type
,
1421 ra_allock(as
, (int32_t)irt_toitype(t
), allow
));
1424 if (ra_hasreg(dest
)) {
1425 if (!LJ_SOFTFP
&& irt_isnum(t
))
1426 emit_hsi(as
, MIPSI_LDC1
, dest
, idx
, ofs
);
1428 emit_tsi(as
, MIPSI_LW
, dest
, idx
, ofs
+(LJ_BE
?4:0));
1430 emit_tsi(as
, MIPSI_LW
, type
, idx
, ofs
+(LJ_BE
?0:4));
1432 if (ra_hasreg(dest
)) {
1433 if (!LJ_SOFTFP
&& irt_isnum(t
)) {
1434 emit_hsi(as
, MIPSI_LDC1
, dest
, idx
, ofs
);
1440 emit_dta(as
, MIPSI_DSRA32
, type
, dest
, 15);
1441 emit_tsi(as
, MIPSI_LD
, dest
, idx
, ofs
);
1445 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
1447 RegSet allow
= RSET_GPR
;
1448 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
1450 if (ir
->r
== RID_SINK
)
1452 if (!LJ_SOFTFP32
&& irt_isnum(ir
->t
)) {
1453 src
= ra_alloc1(as
, ir
->op2
, LJ_SOFTFP
? RSET_GPR
: RSET_FPR
);
1454 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
1455 emit_hsi(as
, LJ_SOFTFP
? MIPSI_SD
: MIPSI_SDC1
, src
, idx
, ofs
);
1458 if (!irt_ispri(ir
->t
)) {
1459 src
= ra_alloc1(as
, ir
->op2
, allow
);
1460 rset_clear(allow
, src
);
1462 if (LJ_SOFTFP
&& (ir
+1)->o
== IR_HIOP
)
1463 type
= ra_alloc1(as
, (ir
+1)->op2
, allow
);
1465 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1466 rset_clear(allow
, type
);
1467 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
1469 emit_tsi(as
, MIPSI_SW
, src
, idx
, ofs
+(LJ_BE
?4:0));
1470 emit_tsi(as
, MIPSI_SW
, type
, idx
, ofs
+(LJ_BE
?0:4));
1473 if (irt_ispri(ir
->t
)) {
1474 tmp
= ra_allock(as
, ~((int64_t)~irt_toitype(ir
->t
) << 47), allow
);
1475 rset_clear(allow
, tmp
);
1477 src
= ra_alloc1(as
, ir
->op2
, allow
);
1478 rset_clear(allow
, src
);
1479 type
= ra_allock(as
, (int64_t)irt_toitype(ir
->t
) << 47, allow
);
1480 rset_clear(allow
, type
);
1482 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
1483 emit_tsi(as
, MIPSI_SD
, tmp
, idx
, ofs
);
1484 if (ra_hasreg(src
)) {
1485 if (irt_isinteger(ir
->t
)) {
1486 emit_dst(as
, MIPSI_DADDU
, tmp
, tmp
, type
);
1487 emit_tsml(as
, MIPSI_DEXT
, tmp
, src
, 31, 0);
1489 emit_dst(as
, MIPSI_DADDU
, tmp
, src
, type
);
1496 static void asm_sload(ASMState
*as
, IRIns
*ir
)
1498 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
1499 RegSet allow
= RSET_GPR
;
1502 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 4 : 0);
1503 int hiop
= (LJ_SOFTFP32
&& (ir
+1)->o
== IR_HIOP
);
1507 int32_t ofs
= 8*((int32_t)ir
->op1
-2);
1509 lj_assertA(!(ir
->op2
& IRSLOAD_PARENT
),
1510 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1511 lj_assertA(irt_isguard(ir
->t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
),
1512 "inconsistent SLOAD variant");
1514 lj_assertA(!(ir
->op2
& IRSLOAD_CONVERT
),
1515 "unsplit SLOAD convert"); /* Handled by LJ_SOFTFP SPLIT. */
1516 if (hiop
&& ra_used(ir
+1)) {
1517 type
= ra_dest(as
, ir
+1, allow
);
1518 rset_clear(allow
, type
);
1521 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
1522 dest
= ra_scratch(as
, LJ_SOFTFP
? allow
: RSET_FPR
);
1523 asm_tointg(as
, ir
, dest
);
1524 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
1528 lj_assertA((LJ_SOFTFP32
? 0 : irt_isnum(ir
->t
)) ||
1529 irt_isint(ir
->t
) || irt_isaddr(ir
->t
),
1530 "bad SLOAD type %d", irt_type(ir
->t
));
1531 dest
= ra_dest(as
, ir
, (!LJ_SOFTFP
&& irt_isnum(t
)) ? RSET_FPR
: allow
);
1532 rset_clear(allow
, dest
);
1533 base
= ra_alloc1(as
, REF_BASE
, allow
);
1534 rset_clear(allow
, base
);
1535 if (!LJ_SOFTFP32
&& (ir
->op2
& IRSLOAD_CONVERT
)) {
1537 Reg tmp
= ra_scratch(as
, LJ_SOFTFP
? RSET_GPR
: RSET_FPR
);
1539 ra_evictset(as
, rset_exclude(RSET_SCRATCH
, dest
));
1540 ra_destreg(as
, ir
, RID_RET
);
1541 emit_call(as
, (void *)lj_ir_callinfo
[IRCALL_softfp_d2i
].func
, 0);
1542 if (tmp
!= REGARG_FIRSTGPR
)
1543 emit_move(as
, REGARG_FIRSTGPR
, tmp
);
1545 emit_tg(as
, MIPSI_MFC1
, dest
, tmp
);
1546 emit_fg(as
, MIPSI_TRUNC_W_D
, tmp
, tmp
);
1549 t
.irt
= IRT_NUM
; /* Check for original type. */
1551 Reg tmp
= ra_scratch(as
, RSET_GPR
);
1553 ra_evictset(as
, rset_exclude(RSET_SCRATCH
, dest
));
1554 ra_destreg(as
, ir
, RID_RET
);
1555 emit_call(as
, (void *)lj_ir_callinfo
[IRCALL_softfp_i2d
].func
, 0);
1556 emit_dta(as
, MIPSI_SLL
, REGARG_FIRSTGPR
, tmp
, 0);
1558 emit_fg(as
, MIPSI_CVT_D_W
, dest
, dest
);
1559 emit_tg(as
, MIPSI_MTC1
, tmp
, dest
);
1562 t
.irt
= IRT_INT
; /* Check for original type. */
1566 else if (irt_isaddr(t
)) {
1567 /* Clear type from pointers. */
1568 emit_tsml(as
, MIPSI_DEXTM
, dest
, dest
, 14, 0);
1569 } else if (irt_isint(t
) && (ir
->op2
& IRSLOAD_TYPECHECK
)) {
1570 /* Sign-extend integers. */
1571 emit_dta(as
, MIPSI_SLL
, dest
, dest
, 0);
1576 base
= ra_alloc1(as
, REF_BASE
, allow
);
1577 rset_clear(allow
, base
);
1580 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1584 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1585 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, type
, (int32_t)LJ_TISNUM
);
1587 Reg ktype
= ra_allock(as
, (ir
->op2
& IRSLOAD_KEYINDEX
) ? LJ_KEYINDEX
: irt_toitype(t
), allow
);
1588 asm_guard(as
, MIPSI_BNE
, type
, ktype
);
1591 if (ra_hasreg(dest
)) {
1592 if (!LJ_SOFTFP
&& irt_isnum(t
))
1593 emit_hsi(as
, MIPSI_LDC1
, dest
, base
, ofs
);
1595 emit_tsi(as
, MIPSI_LW
, dest
, base
, ofs
^ (LJ_BE
?4:0));
1597 if (ra_hasreg(type
))
1598 emit_tsi(as
, MIPSI_LW
, type
, base
, ofs
^ (LJ_BE
?0:4));
1600 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1601 type
= dest
< RID_MAX_GPR
? dest
: RID_TMP
;
1603 asm_guard(as
, MIPSI_BNE
, type
,
1604 ra_allock(as
, ~((int64_t)~irt_toitype(t
) << 47) , allow
));
1605 } else if ((ir
->op2
& IRSLOAD_KEYINDEX
)) {
1606 asm_guard(as
, MIPSI_BNE
, RID_TMP
,
1607 ra_allock(as
, (int32_t)LJ_KEYINDEX
, allow
));
1608 emit_dta(as
, MIPSI_DSRA32
, RID_TMP
, type
, 0);
1611 asm_guard(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1612 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)LJ_TISNUM
);
1613 if (!LJ_SOFTFP
&& ra_hasreg(dest
))
1614 emit_hsi(as
, MIPSI_LDC1
, dest
, base
, ofs
);
1616 asm_guard(as
, MIPSI_BNE
, RID_TMP
,
1617 ra_allock(as
, (int32_t)irt_toitype(t
), allow
));
1619 emit_dta(as
, MIPSI_DSRA32
, RID_TMP
, type
, 15);
1621 emit_tsi(as
, MIPSI_LD
, type
, base
, ofs
);
1622 } else if (ra_hasreg(dest
)) {
1623 if (!LJ_SOFTFP
&& irt_isnum(t
))
1624 emit_hsi(as
, MIPSI_LDC1
, dest
, base
, ofs
);
1626 emit_tsi(as
, irt_isint(t
) ? MIPSI_LW
: MIPSI_LD
, dest
, base
,
1627 ofs
^ ((LJ_BE
&& irt_isint(t
)) ? 4 : 0));
1632 /* -- Allocations --------------------------------------------------------- */
1635 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1637 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1638 CTypeID id
= (CTypeID
)IR(ir
->op1
)->i
;
1640 CTInfo info
= lj_ctype_info(cts
, id
, &sz
);
1641 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1643 RegSet drop
= RSET_SCRATCH
;
1644 lj_assertA(sz
!= CTSIZE_INVALID
|| (ir
->o
== IR_CNEW
&& ir
->op2
!= REF_NIL
),
1645 "bad CNEW/CNEWI operands");
1648 if (ra_hasreg(ir
->r
))
1649 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1650 ra_evictset(as
, drop
);
1652 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1654 /* Initialize immutable cdata object. */
1655 if (ir
->o
== IR_CNEWI
) {
1656 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1658 int32_t ofs
= sizeof(GCcdata
);
1661 lj_assertA((ir
+1)->o
== IR_HIOP
, "expected HIOP for CNEWI");
1665 Reg r
= ra_alloc1z(as
, ir
->op2
, allow
);
1666 emit_tsi(as
, MIPSI_SW
, r
, RID_RET
, ofs
);
1667 rset_clear(allow
, r
);
1668 if (ofs
== sizeof(GCcdata
)) break;
1669 ofs
-= 4; if (LJ_BE
) ir
++; else ir
--;
1672 emit_tsi(as
, sz
== 8 ? MIPSI_SD
: MIPSI_SW
, ra_alloc1(as
, ir
->op2
, allow
),
1673 RID_RET
, sizeof(GCcdata
));
1675 lj_assertA(sz
== 4 || sz
== 8, "bad CNEWI size %d", sz
);
1676 } else if (ir
->op2
!= REF_NIL
) { /* Create VLA/VLS/aligned cdata. */
1677 ci
= &lj_ir_callinfo
[IRCALL_lj_cdata_newv
];
1678 args
[0] = ASMREF_L
; /* lua_State *L */
1679 args
[1] = ir
->op1
; /* CTypeID id */
1680 args
[2] = ir
->op2
; /* CTSize sz */
1681 args
[3] = ASMREF_TMP1
; /* CTSize align */
1682 asm_gencall(as
, ci
, args
);
1683 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP1
), (int32_t)ctype_align(info
));
1687 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1688 emit_tsi(as
, MIPSI_SB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1689 emit_tsi(as
, MIPSI_SH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1690 emit_ti(as
, MIPSI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1691 emit_ti(as
, MIPSI_LI
, RID_TMP
, id
); /* Lower 16 bit used. Sign-ext ok. */
1692 args
[0] = ASMREF_L
; /* lua_State *L */
1693 args
[1] = ASMREF_TMP1
; /* MSize size */
1694 asm_gencall(as
, ci
, args
);
1695 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1696 ra_releasetmp(as
, ASMREF_TMP1
));
1700 /* -- Write barriers ------------------------------------------------------ */
1702 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1704 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1705 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1707 MCLabel l_end
= emit_label(as
);
1708 emit_tsi(as
, MIPSI_AS
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1709 emit_tsi(as
, MIPSI_SB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1710 emit_setgl(as
, tab
, gc
.grayagain
);
1711 emit_getgl(as
, link
, gc
.grayagain
);
1712 emit_dst(as
, MIPSI_XOR
, mark
, mark
, RID_TMP
); /* Clear black bit. */
1713 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1714 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, mark
, LJ_GC_BLACK
);
1715 emit_tsi(as
, MIPSI_LBU
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1718 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1720 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1724 /* No need for other object barriers (yet). */
1725 lj_assertA(IR(ir
->op1
)->o
== IR_UREFC
, "bad OBAR type");
1726 ra_evictset(as
, RSET_SCRATCH
);
1727 l_end
= emit_label(as
);
1728 args
[0] = ASMREF_TMP1
; /* global_State *g */
1729 args
[1] = ir
->op1
; /* TValue *tv */
1730 asm_gencall(as
, ci
, args
);
1731 emit_tsi(as
, MIPSI_AADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1732 obj
= IR(ir
->op1
)->r
;
1733 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1734 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1735 emit_tsi(as
, MIPSI_ANDI
, tmp
, tmp
, LJ_GC_BLACK
);
1736 emit_branch(as
, MIPSI_BEQ
, RID_TMP
, RID_ZERO
, l_end
);
1737 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1738 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1739 emit_tsi(as
, MIPSI_LBU
, tmp
, obj
,
1740 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1741 emit_tsi(as
, MIPSI_LBU
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1744 /* -- Arithmetic and logic operations ------------------------------------- */
1747 static void asm_fparith(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1749 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1750 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1751 right
= (left
>> 8); left
&= 255;
1752 emit_fgh(as
, mi
, dest
, left
, right
);
1755 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, MIPSIns mi
)
1757 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1758 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1759 emit_fg(as
, mi
, dest
, left
);
1764 static void asm_fpmath(ASMState
*as
, IRIns
*ir
)
1767 if (ir
->op2
<= IRFPM_TRUNC
)
1768 asm_callround(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1769 else if (ir
->op2
== IRFPM_SQRT
)
1770 asm_fpunary(as
, ir
, MIPSI_SQRT_D
);
1773 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1778 #define asm_fpadd(as, ir) asm_fparith(as, ir, MIPSI_ADD_D)
1779 #define asm_fpsub(as, ir) asm_fparith(as, ir, MIPSI_SUB_D)
1780 #define asm_fpmul(as, ir) asm_fparith(as, ir, MIPSI_MUL_D)
1781 #elif LJ_64 /* && LJ_SOFTFP */
1782 #define asm_fpadd(as, ir) asm_callid(as, ir, IRCALL_softfp_add)
1783 #define asm_fpsub(as, ir) asm_callid(as, ir, IRCALL_softfp_sub)
1784 #define asm_fpmul(as, ir) asm_callid(as, ir, IRCALL_softfp_mul)
1787 static void asm_add(ASMState
*as
, IRIns
*ir
)
1796 /* TODO MIPSR6: Fuse ADD(BSHL(a,1-4),b) or ADD(ADD(a,a),b) to MIPSI_ALSA. */
1797 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1798 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1799 if (irref_isk(ir
->op2
)) {
1800 intptr_t k
= get_kval(as
, ir
->op2
);
1802 emit_tsi(as
, (LJ_64
&& irt_is64(t
)) ? MIPSI_DADDIU
: MIPSI_ADDIU
, dest
,
1807 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1808 emit_dst(as
, (LJ_64
&& irt_is64(t
)) ? MIPSI_DADDU
: MIPSI_ADDU
, dest
,
1813 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1816 if (irt_isnum(ir
->t
)) {
1821 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1822 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1823 right
= (left
>> 8); left
&= 255;
1824 emit_dst(as
, (LJ_64
&& irt_is64(ir
->t
)) ? MIPSI_DSUBU
: MIPSI_SUBU
, dest
,
1829 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1832 if (irt_isnum(ir
->t
)) {
1837 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1838 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1839 right
= (left
>> 8); left
&= 255;
1840 if (LJ_64
&& irt_is64(ir
->t
)) {
1841 #if !LJ_TARGET_MIPSR6
1842 emit_dst(as
, MIPSI_MFLO
, dest
, 0, 0);
1843 emit_dst(as
, MIPSI_DMULT
, 0, left
, right
);
1845 emit_dst(as
, MIPSI_DMUL
, dest
, left
, right
);
1848 emit_dst(as
, MIPSI_MUL
, dest
, left
, right
);
1854 static void asm_fpdiv(ASMState
*as
, IRIns
*ir
)
1857 asm_fparith(as
, ir
, MIPSI_DIV_D
);
1859 asm_callid(as
, ir
, IRCALL_softfp_div
);
1864 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1867 if (irt_isnum(ir
->t
)) {
1868 asm_fpunary(as
, ir
, MIPSI_NEG_D
);
1870 #elif LJ_64 /* && LJ_SOFTFP */
1871 if (irt_isnum(ir
->t
)) {
1872 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1873 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1874 emit_dst(as
, MIPSI_XOR
, dest
, left
,
1875 ra_allock(as
, 0x8000000000000000ll
, rset_exclude(RSET_GPR
, dest
)));
1879 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1880 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1881 emit_dst(as
, (LJ_64
&& irt_is64(ir
->t
)) ? MIPSI_DSUBU
: MIPSI_SUBU
, dest
,
1887 #define asm_abs(as, ir) asm_fpunary(as, ir, MIPSI_ABS_D)
1888 #elif LJ_64 /* && LJ_SOFTFP */
1889 static void asm_abs(ASMState
*as
, IRIns
*ir
)
1891 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1892 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1893 emit_tsml(as
, MIPSI_DEXTM
, dest
, left
, 30, 0);
1897 static void asm_arithov(ASMState
*as
, IRIns
*ir
)
1899 /* TODO MIPSR6: bovc/bnvc. Caveat: no delay slot to load RID_TMP. */
1900 Reg right
, left
, tmp
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1901 lj_assertA(!irt_is64(ir
->t
), "bad usage");
1902 if (irref_isk(ir
->op2
)) {
1903 int k
= IR(ir
->op2
)->i
;
1904 if (ir
->o
== IR_SUBOV
) k
= (int)(~(unsigned int)k
+1u);
1905 if (checki16(k
)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1906 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1907 asm_guard(as
, k
>= 0 ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
1908 emit_dst(as
, MIPSI_SLT
, RID_TMP
, dest
, dest
== left
? RID_TMP
: left
);
1909 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1910 if (dest
== left
) emit_move(as
, RID_TMP
, left
);
1914 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1915 right
= (left
>> 8); left
&= 255;
1916 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
, left
),
1918 asm_guard(as
, MIPSI_BLTZ
, RID_TMP
, 0);
1919 emit_dst(as
, MIPSI_AND
, RID_TMP
, RID_TMP
, tmp
);
1920 if (ir
->o
== IR_ADDOV
) { /* ((dest^left) & (dest^right)) < 0 */
1921 emit_dst(as
, MIPSI_XOR
, RID_TMP
, dest
, dest
== right
? RID_TMP
: right
);
1922 } else { /* ((dest^left) & (dest^~right)) < 0 */
1923 emit_dst(as
, MIPSI_XOR
, RID_TMP
, RID_TMP
, dest
);
1924 emit_dst(as
, MIPSI_NOR
, RID_TMP
, dest
== right
? RID_TMP
: right
, RID_ZERO
);
1926 emit_dst(as
, MIPSI_XOR
, tmp
, dest
, dest
== left
? RID_TMP
: left
);
1927 emit_dst(as
, ir
->o
== IR_ADDOV
? MIPSI_ADDU
: MIPSI_SUBU
, dest
, left
, right
);
1928 if (dest
== left
|| dest
== right
)
1929 emit_move(as
, RID_TMP
, dest
== left
? left
: right
);
1932 #define asm_addov(as, ir) asm_arithov(as, ir)
1933 #define asm_subov(as, ir) asm_arithov(as, ir)
1935 static void asm_mulov(ASMState
*as
, IRIns
*ir
)
1937 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1938 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1939 right
= (left
>> 8); left
&= 255;
1940 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
, left
),
1942 asm_guard(as
, MIPSI_BNE
, RID_TMP
, tmp
);
1943 emit_dta(as
, MIPSI_SRA
, RID_TMP
, dest
, 31);
1944 #if !LJ_TARGET_MIPSR6
1945 emit_dst(as
, MIPSI_MFHI
, tmp
, 0, 0);
1946 emit_dst(as
, MIPSI_MFLO
, dest
, 0, 0);
1947 emit_dst(as
, MIPSI_MULT
, 0, left
, right
);
1949 emit_dst(as
, MIPSI_MUL
, dest
, left
, right
);
1950 emit_dst(as
, MIPSI_MUH
, tmp
, left
, right
);
1954 #if LJ_32 && LJ_HASFFI
1955 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1957 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1958 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1959 if (irref_isk(ir
->op2
)) {
1960 int32_t k
= IR(ir
->op2
)->i
;
1962 emit_dst(as
, MIPSI_ADDU
, dest
, left
, RID_TMP
);
1964 } else if (checki16(k
)) {
1965 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1966 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1970 emit_dst(as
, MIPSI_ADDU
, dest
, dest
, RID_TMP
);
1971 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1972 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
1975 dest
= ra_dest(as
, ir
, RSET_GPR
);
1976 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1977 if (irref_isk(ir
->op2
)) {
1978 int32_t k
= IR(ir
->op2
)->i
;
1981 emit_move(as
, dest
, left
);
1983 } else if (checki16(k
)) {
1985 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, left
));
1986 emit_move(as
, dest
, tmp
);
1989 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, left
);
1990 emit_tsi(as
, MIPSI_ADDIU
, dest
, left
, k
);
1994 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1995 if (dest
== left
&& dest
== right
) {
1996 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
1997 emit_move(as
, dest
, tmp
);
2000 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, dest
, dest
== left
? right
: left
);
2001 emit_dst(as
, MIPSI_ADDU
, dest
, left
, right
);
2004 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
2006 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2007 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
2008 right
= (left
>> 8); left
&= 255;
2009 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
2010 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
2012 dest
= ra_dest(as
, ir
, RSET_GPR
);
2013 left
= ra_alloc2(as
, ir
, RSET_GPR
);
2014 right
= (left
>> 8); left
&= 255;
2016 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
2017 emit_move(as
, dest
, tmp
);
2020 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, left
, dest
);
2021 emit_dst(as
, MIPSI_SUBU
, dest
, left
, right
);
2024 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
2026 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2027 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
2028 emit_dst(as
, MIPSI_SUBU
, dest
, dest
, RID_TMP
);
2029 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
2031 dest
= ra_dest(as
, ir
, RSET_GPR
);
2032 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
2033 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_ZERO
, dest
);
2034 emit_dst(as
, MIPSI_SUBU
, dest
, RID_ZERO
, left
);
2038 static void asm_bnot(ASMState
*as
, IRIns
*ir
)
2040 Reg left
, right
, dest
= ra_dest(as
, ir
, RSET_GPR
);
2041 IRIns
*irl
= IR(ir
->op1
);
2042 if (mayfuse(as
, ir
->op1
) && irl
->o
== IR_BOR
) {
2043 left
= ra_alloc2(as
, irl
, RSET_GPR
);
2044 right
= (left
>> 8); left
&= 255;
2046 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
2049 emit_dst(as
, MIPSI_NOR
, dest
, left
, right
);
2052 static void asm_bswap(ASMState
*as
, IRIns
*ir
)
2054 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2055 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
2057 if ((as
->flags
& JIT_F_MIPSXXR2
)) {
2058 emit_dta(as
, MIPSI_ROTR
, dest
, RID_TMP
, 16);
2059 emit_dst(as
, MIPSI_WSBH
, RID_TMP
, 0, left
);
2061 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), dest
));
2062 emit_dst(as
, MIPSI_OR
, dest
, dest
, tmp
);
2063 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
2064 emit_tsi(as
, MIPSI_ANDI
, dest
, dest
, 0xff00);
2065 emit_dta(as
, MIPSI_SLL
, RID_TMP
, RID_TMP
, 8);
2066 emit_dta(as
, MIPSI_SRL
, dest
, left
, 8);
2067 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, left
, 0xff00);
2068 emit_dst(as
, MIPSI_OR
, tmp
, tmp
, RID_TMP
);
2069 emit_dta(as
, MIPSI_SRL
, tmp
, left
, 24);
2070 emit_dta(as
, MIPSI_SLL
, RID_TMP
, left
, 24);
2073 if (irt_is64(ir
->t
)) {
2074 emit_dst(as
, MIPSI_DSHD
, dest
, 0, RID_TMP
);
2075 emit_dst(as
, MIPSI_DSBH
, RID_TMP
, 0, left
);
2077 emit_dta(as
, MIPSI_ROTR
, dest
, RID_TMP
, 16);
2078 emit_dst(as
, MIPSI_WSBH
, RID_TMP
, 0, left
);
2083 static void asm_bitop(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
2085 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2086 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
2087 if (irref_isk(ir
->op2
)) {
2088 intptr_t k
= get_kval(as
, ir
->op2
);
2090 emit_tsi(as
, mik
, dest
, left
, k
);
2094 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
2095 emit_dst(as
, mi
, dest
, left
, right
);
2098 #define asm_band(as, ir) asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
2099 #define asm_bor(as, ir) asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
2100 #define asm_bxor(as, ir) asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
2102 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, MIPSIns mi
, MIPSIns mik
)
2104 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2105 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
2106 uint32_t shift
= (uint32_t)IR(ir
->op2
)->i
;
2107 if (LJ_64
&& irt_is64(ir
->t
)) mik
|= (shift
& 32) ? MIPSI_D32
: MIPSI_D
;
2108 emit_dta(as
, mik
, dest
, ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
),
2111 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
2112 right
= (left
>> 8); left
&= 255;
2113 if (LJ_64
&& irt_is64(ir
->t
)) mi
|= MIPSI_DV
;
2114 emit_dst(as
, mi
, dest
, right
, left
); /* Shift amount is in rs. */
2118 #define asm_bshl(as, ir) asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
2119 #define asm_bshr(as, ir) asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
2120 #define asm_bsar(as, ir) asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
2121 #define asm_brol(as, ir) lj_assertA(0, "unexpected BROL")
2123 static void asm_bror(ASMState
*as
, IRIns
*ir
)
2125 if (LJ_64
|| (as
->flags
& JIT_F_MIPSXXR2
)) {
2126 asm_bitshift(as
, ir
, MIPSI_ROTRV
, MIPSI_ROTR
);
2128 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2129 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
2130 uint32_t shift
= (uint32_t)(IR(ir
->op2
)->i
& 31);
2131 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
2132 emit_rotr(as
, dest
, left
, RID_TMP
, shift
);
2134 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
2135 right
= (left
>> 8); left
&= 255;
2136 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
2137 emit_dst(as
, MIPSI_SRLV
, dest
, right
, left
);
2138 emit_dst(as
, MIPSI_SLLV
, RID_TMP
, RID_TMP
, left
);
2139 emit_dst(as
, MIPSI_SUBU
, RID_TMP
, ra_allock(as
, 32, RSET_GPR
), right
);
2145 static void asm_sfpmin_max(ASMState
*as
, IRIns
*ir
)
2147 CCallInfo ci
= lj_ir_callinfo
[(IROp
)ir
->o
== IR_MIN
? IRCALL_lj_vm_sfmin
: IRCALL_lj_vm_sfmax
];
2154 args
[0^LJ_BE
] = ir
->op1
;
2155 args
[1^LJ_BE
] = (ir
+1)->op1
;
2156 args
[2^LJ_BE
] = ir
->op2
;
2157 args
[3^LJ_BE
] = (ir
+1)->op2
;
2159 asm_setupresult(as
, ir
, &ci
);
2160 emit_call(as
, (void *)ci
.func
, 0);
2162 asm_gencall(as
, &ci
, args
);
2166 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
2168 if (!LJ_SOFTFP32
&& irt_isnum(ir
->t
)) {
2170 asm_sfpmin_max(as
, ir
);
2172 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
2173 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
2174 right
= (left
>> 8); left
&= 255;
2175 #if !LJ_TARGET_MIPSR6
2177 emit_fg(as
, MIPSI_MOVF_D
, dest
, right
);
2179 emit_fg(as
, MIPSI_MOVT_D
, dest
, left
);
2180 if (dest
!= right
) emit_fg(as
, MIPSI_MOV_D
, dest
, right
);
2182 emit_fgh(as
, MIPSI_C_OLT_D
, 0, ismax
? right
: left
, ismax
? left
: right
);
2184 emit_fgh(as
, ismax
? MIPSI_MAX_D
: MIPSI_MIN_D
, dest
, left
, right
);
2188 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2189 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
2190 right
= (left
>> 8); left
&= 255;
2191 if (left
== right
) {
2192 if (dest
!= left
) emit_move(as
, dest
, left
);
2194 #if !LJ_TARGET_MIPSR6
2196 emit_dst(as
, MIPSI_MOVN
, dest
, right
, RID_TMP
);
2198 emit_dst(as
, MIPSI_MOVZ
, dest
, left
, RID_TMP
);
2199 if (dest
!= right
) emit_move(as
, dest
, right
);
2202 emit_dst(as
, MIPSI_OR
, dest
, dest
, RID_TMP
);
2203 if (dest
!= right
) {
2204 emit_dst(as
, MIPSI_SELNEZ
, RID_TMP
, right
, RID_TMP
);
2205 emit_dst(as
, MIPSI_SELEQZ
, dest
, left
, RID_TMP
);
2207 emit_dst(as
, MIPSI_SELEQZ
, RID_TMP
, left
, RID_TMP
);
2208 emit_dst(as
, MIPSI_SELNEZ
, dest
, right
, RID_TMP
);
2211 emit_dst(as
, MIPSI_SLT
, RID_TMP
,
2212 ismax
? left
: right
, ismax
? right
: left
);
2217 #define asm_min(as, ir) asm_min_max(as, ir, 0)
2218 #define asm_max(as, ir) asm_min_max(as, ir, 1)
2220 /* -- Comparisons --------------------------------------------------------- */
2223 /* SFP comparisons. */
2224 static void asm_sfpcomp(ASMState
*as
, IRIns
*ir
)
2226 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_softfp_cmp
];
2227 RegSet drop
= RSET_SCRATCH
;
2235 args
[LJ_LE
? 0 : 1] = ir
->op1
; args
[LJ_LE
? 1 : 0] = (ir
+1)->op1
;
2236 args
[LJ_LE
? 2 : 3] = ir
->op2
; args
[LJ_LE
? 3 : 2] = (ir
+1)->op2
;
2239 for (r
= REGARG_FIRSTGPR
; r
<= REGARG_FIRSTGPR
+(LJ_64
?1:3); r
++) {
2240 if (!rset_test(as
->freeset
, r
) &&
2241 regcost_ref(as
->cost
[r
]) == args
[r
-REGARG_FIRSTGPR
])
2242 rset_clear(drop
, r
);
2244 ra_evictset(as
, drop
);
2246 asm_setupresult(as
, ir
, ci
);
2248 switch ((IROp
)ir
->o
) {
2250 asm_guard(as
, MIPSI_BGEZ
, RID_RET
, 0);
2253 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_TMP
);
2254 emit_loadi(as
, RID_TMP
, 1);
2255 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_ZERO
);
2258 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_TMP
);
2259 emit_loadi(as
, RID_TMP
, 2);
2260 asm_guard(as
, MIPSI_BLTZ
, RID_RET
, 0);
2263 asm_guard(as
, MIPSI_BGTZ
, RID_RET
, 0);
2266 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_TMP
);
2267 emit_loadi(as
, RID_TMP
, 2);
2268 asm_guard(as
, MIPSI_BLEZ
, RID_RET
, 0);
2271 asm_guard(as
, MIPSI_BLTZ
, RID_RET
, 0);
2274 asm_guard(as
, MIPSI_BEQ
, RID_RET
, RID_TMP
);
2275 emit_loadi(as
, RID_TMP
, 1);
2277 case IR_UGT
: case IR_ABC
:
2278 asm_guard(as
, MIPSI_BLEZ
, RID_RET
, 0);
2280 case IR_EQ
: case IR_NE
:
2281 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, RID_RET
, RID_ZERO
);
2285 asm_gencall(as
, ci
, args
);
2289 static void asm_comp(ASMState
*as
, IRIns
*ir
)
2291 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
2293 if (!LJ_SOFTFP32
&& irt_isnum(ir
->t
)) {
2295 asm_sfpcomp(as
, ir
);
2297 #if !LJ_TARGET_MIPSR6
2298 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
2299 right
= (left
>> 8); left
&= 255;
2300 asm_guard(as
, (op
&1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
2301 emit_fgh(as
, MIPSI_C_OLT_D
+ ((op
&3) ^ ((op
>>2)&1)), 0, left
, right
);
2303 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
2304 right
= (left
>> 8); left
&= 255;
2305 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_FPR
, left
), right
));
2306 asm_guard(as
, (op
&1) ? MIPSI_BC1NEZ
: MIPSI_BC1EQZ
, 0, (tmp
&31));
2307 emit_fgh(as
, MIPSI_CMP_LT_D
+ ((op
&3) ^ ((op
>>2)&1)), tmp
, left
, right
);
2311 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
2312 if (op
== IR_ABC
) op
= IR_UGT
;
2313 if ((op
&4) == 0 && irref_isk(ir
->op2
) && get_kval(as
, ir
->op2
) == 0) {
2314 MIPSIns mi
= (op
&2) ? ((op
&1) ? MIPSI_BLEZ
: MIPSI_BGTZ
) :
2315 ((op
&1) ? MIPSI_BLTZ
: MIPSI_BGEZ
);
2316 asm_guard(as
, mi
, left
, 0);
2318 if (irref_isk(ir
->op2
)) {
2319 intptr_t k
= get_kval(as
, ir
->op2
);
2322 asm_guard(as
, (op
&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
2323 emit_tsi(as
, (op
&4) ? MIPSI_SLTIU
: MIPSI_SLTI
,
2328 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
2329 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
2330 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
,
2331 RID_TMP
, (op
&2) ? right
: left
, (op
&2) ? left
: right
);
2336 static void asm_equal(ASMState
*as
, IRIns
*ir
)
2338 Reg right
, left
= ra_alloc2(as
, ir
, (!LJ_SOFTFP
&& irt_isnum(ir
->t
)) ?
2339 RSET_FPR
: RSET_GPR
);
2340 right
= (left
>> 8); left
&= 255;
2341 if (!LJ_SOFTFP32
&& irt_isnum(ir
->t
)) {
2343 asm_sfpcomp(as
, ir
);
2344 #elif !LJ_TARGET_MIPSR6
2345 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BC1T
: MIPSI_BC1F
, 0, 0);
2346 emit_fgh(as
, MIPSI_C_EQ_D
, 0, left
, right
);
2348 Reg tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_FPR
, left
), right
));
2349 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BC1NEZ
: MIPSI_BC1EQZ
, 0, (tmp
&31));
2350 emit_fgh(as
, MIPSI_CMP_EQ_D
, tmp
, left
, right
);
2353 asm_guard(as
, (ir
->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, left
, right
);
2357 #if LJ_32 && LJ_HASFFI
2358 /* 64 bit integer comparisons. */
2359 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
2361 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
2362 IROp op
= (ir
-1)->o
;
2364 Reg rightlo
, leftlo
, righthi
, lefthi
= ra_alloc2(as
, ir
, RSET_GPR
);
2365 righthi
= (lefthi
>> 8); lefthi
&= 255;
2366 leftlo
= ra_alloc2(as
, ir
-1,
2367 rset_exclude(rset_exclude(RSET_GPR
, lefthi
), righthi
));
2368 rightlo
= (leftlo
>> 8); leftlo
&= 255;
2369 asm_guard(as
, ((op
^(op
>>1))&1) ? MIPSI_BNE
: MIPSI_BEQ
, RID_TMP
, RID_ZERO
);
2370 l_end
= emit_label(as
);
2371 if (lefthi
!= righthi
)
2372 emit_dst(as
, (op
&4) ? MIPSI_SLTU
: MIPSI_SLT
, RID_TMP
,
2373 (op
&2) ? righthi
: lefthi
, (op
&2) ? lefthi
: righthi
);
2374 emit_dst(as
, MIPSI_SLTU
, RID_TMP
,
2375 (op
&2) ? rightlo
: leftlo
, (op
&2) ? leftlo
: rightlo
);
2376 if (lefthi
!= righthi
)
2377 emit_branch(as
, MIPSI_BEQ
, lefthi
, righthi
, l_end
);
2380 static void asm_comp64eq(ASMState
*as
, IRIns
*ir
)
2382 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
2383 right
= (left
>> 8); left
&= 255;
2384 asm_guard(as
, ((ir
-1)->o
& 1) ? MIPSI_BEQ
: MIPSI_BNE
, RID_TMP
, RID_ZERO
);
2385 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
, left
), right
));
2386 emit_dst(as
, MIPSI_OR
, RID_TMP
, RID_TMP
, tmp
);
2387 emit_dst(as
, MIPSI_XOR
, tmp
, left
, right
);
2388 left
= ra_alloc2(as
, ir
-1, RSET_GPR
);
2389 right
= (left
>> 8); left
&= 255;
2390 emit_dst(as
, MIPSI_XOR
, RID_TMP
, left
, right
);
2394 /* -- Split register ops -------------------------------------------------- */
2396 /* Hiword op of a split 32/32 or 64/64 bit op. Previous op is the loword op. */
2397 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
2399 /* HIOP is marked as a store because it needs its own DCE logic. */
2400 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
2401 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
2402 #if LJ_32 && (LJ_HASFFI || LJ_SOFTFP)
2403 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
2404 as
->curins
--; /* Always skip the CONV. */
2405 #if LJ_HASFFI && !LJ_SOFTFP
2410 } else if ((ir
-1)->o
< IR_EQ
) { /* 64 bit integer comparisons. ORDER IR. */
2411 as
->curins
--; /* Always skip the loword comparison. */
2413 if (!irt_isint(ir
->t
)) {
2414 asm_sfpcomp(as
, ir
-1);
2422 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
2423 as
->curins
--; /* Always skip the loword comparison. */
2425 if (!irt_isint(ir
->t
)) {
2426 asm_sfpcomp(as
, ir
-1);
2431 asm_comp64eq(as
, ir
);
2435 } else if ((ir
-1)->o
== IR_MIN
|| (ir
-1)->o
== IR_MAX
) {
2436 as
->curins
--; /* Always skip the loword min/max. */
2438 asm_sfpmin_max(as
, ir
-1);
2441 } else if ((ir
-1)->o
== IR_XSTORE
) {
2442 as
->curins
--; /* Handle both stores here. */
2443 if ((ir
-1)->r
!= RID_SINK
) {
2444 asm_xstore_(as
, ir
, LJ_LE
? 4 : 0);
2445 asm_xstore_(as
, ir
-1, LJ_LE
? 0 : 4);
2450 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
2451 switch ((ir
-1)->o
) {
2452 #if LJ_32 && LJ_HASFFI
2453 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
2454 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
2455 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
2457 /* Nothing to do here. Handled by lo op itself. */
2460 #if LJ_32 && LJ_SOFTFP
2461 case IR_SLOAD
: case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
2464 ra_allocref(as
, ir
->op1
, RSET_GPR
); /* Mark lo op as used. */
2466 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: case IR_TOSTR
: case IR_TMPREF
:
2467 /* Nothing to do here. Handled by lo op itself. */
2470 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: case IR_CALLXS
:
2472 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
2474 default: lj_assertA(0, "bad HIOP for op %d", (ir
-1)->o
); break;
2478 /* -- Profiling ----------------------------------------------------------- */
2480 static void asm_prof(ASMState
*as
, IRIns
*ir
)
2483 asm_guard(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
);
2484 emit_tsi(as
, MIPSI_ANDI
, RID_TMP
, RID_TMP
, HOOK_PROFILE
);
2485 emit_lsglptr(as
, MIPSI_LBU
, RID_TMP
,
2486 (int32_t)offsetof(global_State
, hookmask
));
2489 /* -- Stack handling ------------------------------------------------------ */
2491 /* Check Lua stack size for overflow. Use exit handler as fallback. */
2492 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
2493 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
2495 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
2496 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
2497 ExitNo oldsnap
= as
->snapno
;
2498 rset_clear(allow
, pbase
);
2500 tmp
= allow
? rset_pickbot(allow
) :
2501 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
2503 tmp
= allow
? rset_pickbot(allow
) : RID_RET
;
2505 as
->snapno
= exitno
;
2506 asm_guard(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
);
2507 as
->snapno
= oldsnap
;
2508 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
2509 emit_tsi(as
, MIPSI_AL
, tmp
, RID_SP
, 0);
2511 ra_modified(as
, tmp
);
2512 emit_tsi(as
, MIPSI_SLTIU
, RID_TMP
, RID_TMP
, (int32_t)(8*topslot
));
2513 emit_dst(as
, MIPSI_ASUBU
, RID_TMP
, tmp
, pbase
);
2514 emit_tsi(as
, MIPSI_AL
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
2515 if (pbase
== RID_TMP
)
2516 emit_getgl(as
, RID_TMP
, jit_base
);
2517 emit_getgl(as
, tmp
, cur_L
);
2518 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
2519 emit_tsi(as
, MIPSI_AS
, tmp
, RID_SP
, 0);
2522 /* Restore Lua stack from on-trace state. */
2523 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
2525 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
2526 #if LJ_32 || defined(LUA_USE_ASSERT)
2527 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1-LJ_FR2
];
2529 MSize n
, nent
= snap
->nent
;
2530 /* Store the value of all modified slots to the Lua stack. */
2531 for (n
= 0; n
< nent
; n
++) {
2532 SnapEntry sn
= map
[n
];
2533 BCReg s
= snap_slot(sn
);
2534 int32_t ofs
= 8*((int32_t)s
-1-LJ_FR2
);
2535 IRRef ref
= snap_ref(sn
);
2536 IRIns
*ir
= IR(ref
);
2537 if ((sn
& SNAP_NORESTORE
))
2539 if (irt_isnum(ir
->t
)) {
2542 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
2543 /* LJ_SOFTFP: must be a number constant. */
2544 lj_assertA(irref_isk(ref
), "unsplit FP op");
2545 tmp
= ra_allock(as
, (int32_t)ir_knum(ir
)->u32
.lo
, allow
);
2546 emit_tsi(as
, MIPSI_SW
, tmp
, RID_BASE
, ofs
+(LJ_BE
?4:0));
2547 if (rset_test(as
->freeset
, tmp
+1)) allow
= RID2RSET(tmp
+1);
2548 tmp
= ra_allock(as
, (int32_t)ir_knum(ir
)->u32
.hi
, allow
);
2549 emit_tsi(as
, MIPSI_SW
, tmp
, RID_BASE
, ofs
+(LJ_BE
?0:4));
2550 #elif LJ_SOFTFP /* && LJ_64 */
2551 Reg src
= ra_alloc1(as
, ref
, rset_exclude(RSET_GPR
, RID_BASE
));
2552 emit_tsi(as
, MIPSI_SD
, src
, RID_BASE
, ofs
);
2554 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
2555 emit_hsi(as
, MIPSI_SDC1
, src
, RID_BASE
, ofs
);
2559 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
2561 lj_assertA(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
),
2562 "restore of IR type %d", irt_type(ir
->t
));
2563 if (!irt_ispri(ir
->t
)) {
2564 Reg src
= ra_alloc1(as
, ref
, allow
);
2565 rset_clear(allow
, src
);
2566 emit_tsi(as
, MIPSI_SW
, src
, RID_BASE
, ofs
+(LJ_BE
?4:0));
2568 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
2569 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
2570 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
2572 } else if ((sn
& SNAP_SOFTFPNUM
)) {
2573 type
= ra_alloc1(as
, ref
+1, rset_exclude(RSET_GPR
, RID_BASE
));
2575 } else if ((sn
& SNAP_KEYINDEX
)) {
2576 type
= ra_allock(as
, (int32_t)LJ_KEYINDEX
, allow
);
2578 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
2580 emit_tsi(as
, MIPSI_SW
, type
, RID_BASE
, ofs
+(LJ_BE
?0:4));
2582 if ((sn
& SNAP_KEYINDEX
)) {
2583 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
2584 int64_t kki
= (int64_t)LJ_KEYINDEX
<< 32;
2585 if (irref_isk(ref
)) {
2586 emit_tsi(as
, MIPSI_SD
,
2587 ra_allock(as
, kki
| (int64_t)(uint32_t)ir
->i
, allow
),
2590 Reg src
= ra_alloc1(as
, ref
, allow
);
2591 Reg rki
= ra_allock(as
, kki
, rset_exclude(allow
, src
));
2592 emit_tsi(as
, MIPSI_SD
, RID_TMP
, RID_BASE
, ofs
);
2593 emit_dst(as
, MIPSI_DADDU
, RID_TMP
, src
, rki
);
2596 asm_tvstore64(as
, RID_BASE
, ofs
, ref
);
2602 lj_assertA(map
+ nent
== flinks
, "inconsistent frames in snapshot");
2605 /* -- GC handling --------------------------------------------------------- */
2607 /* Marker to prevent patching the GC check exit. */
2608 #define MIPS_NOPATCH_GC_CHECK MIPSI_OR
2610 /* Check GC threshold and do one or more GC steps. */
2611 static void asm_gc_check(ASMState
*as
)
2613 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
2617 ra_evictset(as
, RSET_SCRATCH
);
2618 l_end
= emit_label(as
);
2619 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2620 /* Assumes asm_snap_prep() already done. */
2621 asm_guard(as
, MIPSI_BNE
, RID_RET
, RID_ZERO
);
2622 args
[0] = ASMREF_TMP1
; /* global_State *g */
2623 args
[1] = ASMREF_TMP2
; /* MSize steps */
2624 asm_gencall(as
, ci
, args
);
2625 l_end
[-3] = MIPS_NOPATCH_GC_CHECK
; /* Replace the nop after the call. */
2626 emit_tsi(as
, MIPSI_AADDIU
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
2627 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
2628 emit_loadi(as
, tmp
, as
->gcsteps
);
2629 /* Jump around GC step if GC total < GC threshold. */
2630 emit_branch(as
, MIPSI_BNE
, RID_TMP
, RID_ZERO
, l_end
);
2631 emit_dst(as
, MIPSI_SLTU
, RID_TMP
, RID_TMP
, tmp
);
2632 emit_getgl(as
, tmp
, gc
.threshold
);
2633 emit_getgl(as
, RID_TMP
, gc
.total
);
2638 /* -- Loop handling ------------------------------------------------------- */
2640 /* Fixup the loop branch. */
2641 static void asm_loop_fixup(ASMState
*as
)
2643 MCode
*p
= as
->mctop
;
2644 MCode
*target
= as
->mcp
;
2646 if (as
->loopinv
) { /* Inverted loop branch? */
2647 /* asm_guard already inverted the cond branch. Only patch the target. */
2648 p
[-3] |= ((target
-p
+2) & 0x0000ffffu
);
2650 p
[-2] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
2654 /* Fixup the tail of the loop. */
2655 static void asm_loop_tail_fixup(ASMState
*as
)
2657 if (as
->loopinv
) as
->mctop
--;
2660 /* -- Head of trace ------------------------------------------------------- */
2662 /* Coalesce BASE register for a root trace. */
2663 static void asm_head_root_base(ASMState
*as
)
2665 IRIns
*ir
= IR(REF_BASE
);
2669 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
2670 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2672 emit_move(as
, r
, RID_BASE
);
2676 /* Coalesce BASE register for a side trace. */
2677 static Reg
asm_head_side_base(ASMState
*as
, IRIns
*irp
)
2679 IRIns
*ir
= IR(REF_BASE
);
2683 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
2684 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2686 return r
; /* Same BASE register already coalesced. */
2687 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
2688 emit_move(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
2691 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
2697 /* -- Tail of trace ------------------------------------------------------- */
2699 /* Fixup the tail code. */
2700 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
2702 MCode
*target
= lnk
? traceref(as
->J
,lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
2703 int32_t spadj
= as
->T
->spadjust
;
2704 MCode
*p
= as
->mctop
-1;
2705 *p
= spadj
? (MIPSI_AADDIU
|MIPSF_T(RID_SP
)|MIPSF_S(RID_SP
)|spadj
) : MIPSI_NOP
;
2706 p
[-1] = MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
2709 /* Prepare tail of code. */
2710 static void asm_tail_prep(ASMState
*as
)
2712 as
->mcp
= as
->mctop
-2; /* Leave room for branch plus nop or stack adj. */
2713 as
->invmcp
= as
->loopref
? as
->mcp
: NULL
;
2716 /* -- Trace setup --------------------------------------------------------- */
2718 /* Ensure there are enough stack slots for call arguments. */
2719 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
2721 IRRef args
[CCI_NARGS_MAX
*2];
2722 uint32_t i
, nargs
= CCI_XNARGS(ci
);
2724 int nslots
= 4, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
2726 int nslots
= 0, ngpr
= REGARG_NUMGPR
;
2728 asm_collectargs(as
, ir
, ci
, args
);
2729 for (i
= 0; i
< nargs
; i
++) {
2731 if (!LJ_SOFTFP
&& args
[i
] && irt_isfp(IR(args
[i
])->t
) &&
2732 nfpr
> 0 && !(ci
->flags
& CCI_VARARG
)) {
2734 ngpr
-= irt_isnum(IR(args
[i
])->t
) ? 2 : 1;
2735 } else if (!LJ_SOFTFP
&& args
[i
] && irt_isnum(IR(args
[i
])->t
)) {
2738 if (ngpr
> 0) ngpr
-= 2; else nslots
= (nslots
+3) & ~1;
2741 if (ngpr
> 0) ngpr
--; else nslots
++;
2744 if (ngpr
> 0) ngpr
--; else nslots
+= 2;
2747 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
2748 as
->evenspill
= nslots
;
2749 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
2752 static void asm_setup_target(ASMState
*as
)
2754 asm_sparejump_setup(as
);
2755 asm_exitstub_setup(as
);
2758 /* -- Trace patching ------------------------------------------------------ */
2760 /* Patch exit jumps of existing machine code to a new target. */
2761 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
2763 MCode
*p
= T
->mcode
;
2764 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
2765 MCode
*px
= exitstub_trace_addr(T
, exitno
);
2766 MCode
*cstart
= NULL
, *cstop
= NULL
;
2767 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
2768 MCode exitload
= MIPSI_LI
| MIPSF_T(RID_TMP
) | exitno
;
2769 MCode tjump
= MIPSI_J
|(((uintptr_t)target
>>2)&0x03ffffffu
);
2770 for (p
++; p
< pe
; p
++) {
2771 if (*p
== exitload
) { /* Look for load of exit number. */
2772 /* Look for exitstub branch. Yes, this covers all used branch variants. */
2773 if (((p
[-1] ^ (px
-p
)) & 0xffffu
) == 0 &&
2774 ((p
[-1] & 0xf0000000u
) == MIPSI_BEQ
||
2775 (p
[-1] & 0xfc1e0000u
) == MIPSI_BLTZ
||
2776 #if !LJ_TARGET_MIPSR6
2777 (p
[-1] & 0xffe00000u
) == MIPSI_BC1F
2779 (p
[-1] & 0xff600000u
) == MIPSI_BC1EQZ
2781 ) && p
[-2] != MIPS_NOPATCH_GC_CHECK
) {
2782 ptrdiff_t delta
= target
- p
;
2783 if (((delta
+ 0x8000) >> 16) == 0) { /* Patch in-range branch. */
2785 p
[-1] = (p
[-1] & 0xffff0000u
) | (delta
& 0xffffu
);
2786 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
2788 if (!cstart
) cstart
= p
-1;
2789 } else { /* Branch out of range. Use spare jump slot in mcarea. */
2790 MCode
*mcjump
= asm_sparejump_use(mcarea
, tjump
);
2792 lj_mcode_sync(mcjump
, mcjump
+1);
2794 if (((delta
+ 0x8000) >> 16) == 0) {
2797 lj_assertJ(0, "spare jump out of range: -Osizemcode too big");
2800 /* Ignore jump slot overflow. Child trace is simply not attached. */
2802 } else if (p
+1 == pe
) {
2803 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
2804 lj_assertJ(p
[1] == MIPSI_NOP
, "expected NOP");
2806 *p
= MIPSI_NOP
; /* Replace the load of the exit number. */
2808 if (!cstart
) cstart
= p
+1;
2812 if (cstart
) lj_mcode_sync(cstart
, cstop
);
2813 lj_mcode_patch(J
, mcarea
, 1);