2 ** PPC IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate two source registers for three-operand instructions. */
22 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
24 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
25 Reg left
= irl
->r
, right
= irr
->r
;
26 if (ra_hasreg(left
)) {
29 right
= ra_allocref(as
, ir
->op2
, rset_exclude(allow
, left
));
32 } else if (ra_hasreg(right
)) {
34 left
= ra_allocref(as
, ir
->op1
, rset_exclude(allow
, right
));
35 } else if (ra_hashint(right
)) {
36 right
= ra_allocref(as
, ir
->op2
, allow
);
37 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(allow
, right
));
39 left
= ra_allocref(as
, ir
->op1
, allow
);
40 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(allow
, left
));
42 return left
| (right
<< 8);
45 /* -- Guard handling ------------------------------------------------------ */
47 /* Setup exit stubs after the end of each trace. */
48 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
51 MCode
*mxp
= as
->mctop
;
52 /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
53 for (i
= nexits
-1; (int32_t)i
>= 0; i
--)
54 *--mxp
= PPCI_BL
|(((-3-i
)&0x00ffffffu
)<<2);
55 *--mxp
= PPCI_LI
|PPCF_T(RID_TMP
)|as
->T
->traceno
; /* Read by exit handler. */
57 *mxp
= PPCI_BL
|((((MCode
*)(void *)lj_vm_exit_handler
-mxp
)&0x00ffffffu
)<<2);
58 *--mxp
= PPCI_MFLR
|PPCF_T(RID_TMP
);
62 static MCode
*asm_exitstub_addr(ASMState
*as
, ExitNo exitno
)
64 /* Keep this in-sync with exitstub_trace_addr(). */
65 return as
->mctop
+ exitno
+ 3;
68 /* Emit conditional branch to exit for guard. */
69 static void asm_guardcc(ASMState
*as
, PPCCC cc
)
71 MCode
*target
= asm_exitstub_addr(as
, as
->snapno
);
73 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
75 *p
= PPCI_B
| (((target
-p
) & 0x00ffffffu
) << 2);
76 emit_condbranch(as
, PPCI_BC
, cc
^4, p
);
79 emit_condbranch(as
, PPCI_BC
, cc
, target
);
82 /* -- Operand fusion ------------------------------------------------------ */
84 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
85 #define CONFLICT_SEARCH_LIM 31
87 /* Check if there's no conflicting instruction between curins and ref. */
88 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
92 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
93 return 0; /* Give up, ref is too far away. */
95 if (ir
[i
].o
== conflict
)
96 return 0; /* Conflict found. */
97 return 1; /* Ok, no conflict. */
100 /* Fuse the array base of colocated arrays. */
101 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
104 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
105 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
106 return (int32_t)sizeof(GCtab
);
110 /* Indicates load/store indexed is ok. */
111 #define AHUREF_LSX ((int32_t)0x80000000)
113 /* Fuse array/hash/upvalue reference into register+offset operand. */
114 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
117 if (ra_noreg(ir
->r
)) {
118 if (ir
->o
== IR_AREF
) {
119 if (mayfuse(as
, ref
)) {
120 if (irref_isk(ir
->op2
)) {
121 IRRef tab
= IR(ir
->op1
)->op1
;
122 int32_t ofs
= asm_fuseabase(as
, tab
);
123 IRRef refa
= ofs
? tab
: ir
->op1
;
124 ofs
+= 8*IR(ir
->op2
)->i
;
127 return ra_alloc1(as
, refa
, allow
);
130 if (*ofsp
== AHUREF_LSX
) {
131 Reg base
= ra_alloc1(as
, ir
->op1
, allow
);
132 Reg idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
133 return base
| (idx
<< 8);
136 } else if (ir
->o
== IR_HREFK
) {
137 if (mayfuse(as
, ref
)) {
138 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
141 return ra_alloc1(as
, ir
->op1
, allow
);
144 } else if (ir
->o
== IR_UREFC
) {
145 if (irref_isk(ir
->op1
)) {
146 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
147 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
148 int32_t jgl
= (intptr_t)J2G(as
->J
);
149 if ((uint32_t)(ofs
-jgl
) < 65536) {
150 *ofsp
= ofs
-jgl
-32768;
153 *ofsp
= (int16_t)ofs
;
154 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
160 return ra_alloc1(as
, ref
, allow
);
163 /* Fuse XLOAD/XSTORE reference into load/store operand. */
164 static void asm_fusexref(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
165 RegSet allow
, int32_t ofs
)
169 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
170 if (ir
->o
== IR_ADD
) {
172 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
175 } else if (ofs
== 0) {
176 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
177 right
= (left
>> 8); left
&= 255;
178 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
181 } else if (ir
->o
== IR_STRREF
) {
182 lua_assert(ofs
== 0);
183 ofs
= (int32_t)sizeof(GCstr
);
184 if (irref_isk(ir
->op2
)) {
185 ofs
+= IR(ir
->op2
)->i
;
187 } else if (irref_isk(ir
->op1
)) {
188 ofs
+= IR(ir
->op1
)->i
;
191 /* NYI: Fuse ADD with constant. */
192 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, allow
);
193 right
= (left
>> 8); left
&= 255;
194 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(allow
, left
), right
));
195 emit_fai(as
, pi
, rt
, tmp
, ofs
);
196 emit_tab(as
, PPCI_ADD
, tmp
, left
, right
);
199 if (!checki16(ofs
)) {
200 Reg left
= ra_alloc1(as
, ref
, allow
);
201 Reg right
= ra_allock(as
, ofs
, rset_exclude(allow
, left
));
202 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
207 base
= ra_alloc1(as
, ref
, allow
);
208 emit_fai(as
, pi
, rt
, base
, ofs
);
211 /* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
212 static void asm_fusexrefx(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
215 IRIns
*ira
= IR(ref
);
217 if (canfuse(as
, ira
) && ira
->o
== IR_ADD
&& ra_noreg(ira
->r
)) {
218 left
= ra_alloc2(as
, ira
, allow
);
219 right
= (left
>> 8); left
&= 255;
221 right
= ra_alloc1(as
, ref
, allow
);
224 emit_tab(as
, pi
, rt
, left
, right
);
227 /* Fuse to multiply-add/sub instruction. */
228 static int asm_fusemadd(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pir
)
230 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
233 ((mayfuse(as
, lref
) && (irm
= IR(lref
), irm
->o
== IR_MUL
) &&
235 (mayfuse(as
, rref
) && (irm
= IR(rref
), irm
->o
== IR_MUL
) &&
236 (rref
= lref
, pi
= pir
, ra_noreg(irm
->r
))))) {
237 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
238 Reg add
= ra_alloc1(as
, rref
, RSET_FPR
);
239 Reg right
, left
= ra_alloc2(as
, irm
, rset_exclude(RSET_FPR
, add
));
240 right
= (left
>> 8); left
&= 255;
241 emit_facb(as
, pi
, dest
, left
, right
, add
);
247 /* -- Calls --------------------------------------------------------------- */
249 /* Generate a call to a C function. */
250 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
252 uint32_t n
, nargs
= CCI_NARGS(ci
);
254 Reg gpr
= REGARG_FIRSTGPR
, fpr
= REGARG_FIRSTFPR
;
255 if ((void *)ci
->func
)
256 emit_call(as
, (void *)ci
->func
);
257 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
261 if (irt_isfp(ir
->t
)) {
262 if (fpr
<= REGARG_LASTFPR
) {
263 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
264 ra_leftov(as
, fpr
, ref
);
267 Reg r
= ra_alloc1(as
, ref
, RSET_FPR
);
268 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
269 emit_spstore(as
, ir
, r
, ofs
);
270 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
273 if (gpr
<= REGARG_LASTGPR
) {
274 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
275 ra_leftov(as
, gpr
, ref
);
278 Reg r
= ra_alloc1(as
, ref
, RSET_GPR
);
279 emit_spstore(as
, ir
, r
, ofs
);
284 if (gpr
<= REGARG_LASTGPR
)
290 if ((ci
->flags
& CCI_VARARG
)) /* Vararg calls need to know about FPR use. */
291 emit_tab(as
, fpr
== REGARG_FIRSTFPR
? PPCI_CRXOR
: PPCI_CREQV
, 6, 6, 6);
294 /* Setup result reg/sp for call. Evict scratch regs. */
295 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
297 RegSet drop
= RSET_SCRATCH
;
298 int hiop
= ((ir
+1)->o
== IR_HIOP
);
299 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
301 if (ra_hasreg(ir
->r
))
302 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
303 if (hiop
&& ra_hasreg((ir
+1)->r
))
304 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
305 ra_evictset(as
, drop
); /* Evictions must be performed first. */
307 lua_assert(!irt_ispri(ir
->t
));
308 if (irt_isfp(ir
->t
)) {
309 if ((ci
->flags
& CCI_CASTU64
)) {
310 /* Use spill slot or temp slots. */
311 int32_t ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
313 if (ra_hasreg(dest
)) {
315 ra_modified(as
, dest
);
316 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, ofs
);
318 emit_tai(as
, PPCI_STW
, RID_RETHI
, RID_SP
, ofs
);
319 emit_tai(as
, PPCI_STW
, RID_RETLO
, RID_SP
, ofs
+4);
321 ra_destreg(as
, ir
, RID_FPRET
);
326 ra_destreg(as
, ir
, RID_RET
);
331 static void asm_call(ASMState
*as
, IRIns
*ir
)
333 IRRef args
[CCI_NARGS_MAX
];
334 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
335 asm_collectargs(as
, ir
, ci
, args
);
336 asm_setupresult(as
, ir
, ci
);
337 asm_gencall(as
, ci
, args
);
340 static void asm_callx(ASMState
*as
, IRIns
*ir
)
342 IRRef args
[CCI_NARGS_MAX
];
346 ci
.flags
= asm_callx_flags(as
, ir
);
347 asm_collectargs(as
, ir
, &ci
, args
);
348 asm_setupresult(as
, ir
, &ci
);
349 func
= ir
->op2
; irf
= IR(func
);
350 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
351 if (irref_isk(func
)) { /* Call to constant address. */
352 ci
.func
= (ASMFunction
)(void *)(irf
->i
);
353 } else { /* Need a non-argument register for indirect calls. */
354 RegSet allow
= RSET_GPR
& ~RSET_RANGE(RID_R0
, REGARG_LASTGPR
+1);
355 Reg freg
= ra_alloc1(as
, func
, allow
);
356 *--as
->mcp
= PPCI_BCTRL
;
357 *--as
->mcp
= PPCI_MTCTR
| PPCF_T(freg
);
358 ci
.func
= (ASMFunction
)(void *)0;
360 asm_gencall(as
, &ci
, args
);
363 static void asm_callid(ASMState
*as
, IRIns
*ir
, IRCallID id
)
365 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
369 asm_setupresult(as
, ir
, ci
);
370 asm_gencall(as
, ci
, args
);
373 /* -- Returns ------------------------------------------------------------- */
375 /* Return to lower frame. Guard that it goes to the right spot. */
376 static void asm_retf(ASMState
*as
, IRIns
*ir
)
378 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
379 void *pc
= ir_kptr(IR(ir
->op2
));
380 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
381 as
->topslot
-= (BCReg
)delta
;
382 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
383 emit_setgl(as
, base
, jit_base
);
384 emit_addptr(as
, base
, -8*delta
);
385 asm_guardcc(as
, CC_NE
);
386 emit_ab(as
, PPCI_CMPW
, RID_TMP
,
387 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
388 emit_tai(as
, PPCI_LWZ
, RID_TMP
, base
, -8);
391 /* -- Type conversions ---------------------------------------------------- */
393 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
395 RegSet allow
= RSET_FPR
;
396 Reg tmp
= ra_scratch(as
, rset_clear(allow
, left
));
397 Reg fbias
= ra_scratch(as
, rset_clear(allow
, tmp
));
398 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
399 Reg hibias
= ra_allock(as
, 0x43300000, rset_exclude(RSET_GPR
, dest
));
400 asm_guardcc(as
, CC_NE
);
401 emit_fab(as
, PPCI_FCMPU
, 0, tmp
, left
);
402 emit_fab(as
, PPCI_FSUB
, tmp
, tmp
, fbias
);
403 emit_fai(as
, PPCI_LFD
, tmp
, RID_SP
, SPOFS_TMP
);
404 emit_tai(as
, PPCI_STW
, RID_TMP
, RID_SP
, SPOFS_TMPLO
);
405 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
406 emit_asi(as
, PPCI_XORIS
, RID_TMP
, dest
, 0x8000);
407 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
408 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
409 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
411 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
412 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
415 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
417 RegSet allow
= RSET_FPR
;
418 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
419 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
420 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
421 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
422 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
423 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
424 emit_fab(as
, PPCI_FADD
, tmp
, left
, right
);
427 static void asm_conv(ASMState
*as
, IRIns
*ir
)
429 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
430 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
431 IRRef lref
= ir
->op1
;
432 lua_assert(irt_type(ir
->t
) != st
);
433 lua_assert(!(irt_isint64(ir
->t
) ||
434 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
435 if (irt_isfp(ir
->t
)) {
436 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
437 if (stfp
) { /* FP to FP conversion. */
438 if (st
== IRT_NUM
) /* double -> float conversion. */
439 emit_fb(as
, PPCI_FRSP
, dest
, ra_alloc1(as
, lref
, RSET_FPR
));
440 else /* float -> double conversion is a no-op on PPC. */
441 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
442 } else { /* Integer to FP conversion. */
443 /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
444 /* IRT_U32: Bias with 2^52, subtract 2^52. */
445 RegSet allow
= RSET_GPR
;
446 Reg left
= ra_alloc1(as
, lref
, allow
);
447 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, left
));
448 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
450 if (irt_isfloat(ir
->t
)) emit_fb(as
, PPCI_FRSP
, dest
, dest
);
451 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
452 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
453 kbias
= (const float *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000));
454 if (st
== IRT_U32
) kbias
++;
455 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31), (void *)kbias
,
456 rset_clear(allow
, hibias
));
457 emit_tai(as
, PPCI_STW
, st
== IRT_U32
? left
: RID_TMP
,
458 RID_SP
, SPOFS_TMPLO
);
459 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
460 if (st
!= IRT_U32
) emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, 0x8000);
462 } else if (stfp
) { /* FP to integer conversion. */
463 if (irt_isguard(ir
->t
)) {
464 /* Checked conversions are only supported from number to int. */
465 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
466 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
468 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
469 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
470 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
471 if (irt_isu32(ir
->t
)) {
472 /* Convert both x and x-2^31 to int and merge results. */
473 Reg tmpi
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
474 emit_asb(as
, PPCI_OR
, dest
, dest
, tmpi
); /* Select with mask idiom. */
475 emit_asb(as
, PPCI_AND
, tmpi
, tmpi
, RID_TMP
);
476 emit_asb(as
, PPCI_ANDC
, dest
, dest
, RID_TMP
);
477 emit_tai(as
, PPCI_LWZ
, tmpi
, RID_SP
, SPOFS_TMPLO
); /* tmp = (int)(x) */
478 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, 0x8000); /* dest += 2^31 */
479 emit_asb(as
, PPCI_SRAWI
, RID_TMP
, dest
, 31); /* mask = -(dest < 0) */
480 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
481 emit_tai(as
, PPCI_LWZ
, dest
,
482 RID_SP
, SPOFS_TMPLO
); /* dest = (int)(x-2^31) */
483 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
484 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
485 emit_fb(as
, PPCI_FCTIWZ
, tmp
, tmp
);
486 emit_fab(as
, PPCI_FSUB
, tmp
, left
, tmp
);
487 emit_lsptr(as
, PPCI_LFS
, (tmp
& 31),
488 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,00000000)),
491 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
492 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
493 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
497 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
498 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
499 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
500 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
501 if ((ir
->op2
& IRCONV_SEXT
))
502 emit_as(as
, st
== IRT_I8
? PPCI_EXTSB
: PPCI_EXTSH
, dest
, left
);
504 emit_rot(as
, PPCI_RLWINM
, dest
, left
, 0, st
== IRT_U8
? 24 : 16, 31);
505 } else { /* 32/64 bit integer conversions. */
506 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
507 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
513 static void asm_conv64(ASMState
*as
, IRIns
*ir
)
515 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
516 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
521 args
[1] = (ir
-1)->op1
;
522 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) {
523 id
= IRCALL_fp64_d2l
+ ((st
== IRT_FLOAT
) ? 2 : 0) + (dt
- IRT_I64
);
526 id
= IRCALL_fp64_l2d
+ ((dt
== IRT_FLOAT
) ? 2 : 0) + (st
- IRT_I64
);
528 ci
= &lj_ir_callinfo
[id
];
529 asm_setupresult(as
, ir
, ci
);
530 asm_gencall(as
, ci
, args
);
534 static void asm_strto(ASMState
*as
, IRIns
*ir
)
536 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
539 RegSet drop
= RSET_SCRATCH
;
540 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
541 ra_evictset(as
, drop
);
542 asm_guardcc(as
, CC_EQ
);
543 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0); /* Test return status. */
544 args
[0] = ir
->op1
; /* GCstr *str */
545 args
[1] = ASMREF_TMP1
; /* TValue *n */
546 asm_gencall(as
, ci
, args
);
547 /* Store the result to the spill slot or temp slots. */
548 ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
549 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_SP
, ofs
);
552 /* Get pointer to TValue. */
553 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
556 if (irt_isnum(ir
->t
)) {
557 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
558 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
559 else /* Otherwise force a spill and use the spill slot. */
560 emit_tai(as
, PPCI_ADDI
, dest
, RID_SP
, ra_spill(as
, ir
));
562 /* Otherwise use g->tmptv to hold the TValue. */
563 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
565 emit_tai(as
, PPCI_ADDI
, dest
, RID_JGL
, offsetof(global_State
, tmptv
)-32768);
566 if (!irt_ispri(ir
->t
)) {
567 Reg src
= ra_alloc1(as
, ref
, allow
);
568 emit_setgl(as
, src
, tmptv
.gcr
);
570 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
571 emit_setgl(as
, type
, tmptv
.it
);
575 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
580 if (irt_isnum(IR(ir
->op1
)->t
) || (ir
+1)->o
== IR_HIOP
) {
581 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
582 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
583 asm_setupresult(as
, ir
, ci
); /* GCstr * */
584 asm_gencall(as
, ci
, args
);
585 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op1
);
587 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
588 args
[1] = ir
->op1
; /* int32_t k */
589 asm_setupresult(as
, ir
, ci
); /* GCstr * */
590 asm_gencall(as
, ci
, args
);
594 /* -- Memory references --------------------------------------------------- */
596 static void asm_aref(ASMState
*as
, IRIns
*ir
)
598 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
600 if (irref_isk(ir
->op2
)) {
601 IRRef tab
= IR(ir
->op1
)->op1
;
602 int32_t ofs
= asm_fuseabase(as
, tab
);
603 IRRef refa
= ofs
? tab
: ir
->op1
;
604 ofs
+= 8*IR(ir
->op2
)->i
;
606 base
= ra_alloc1(as
, refa
, RSET_GPR
);
607 emit_tai(as
, PPCI_ADDI
, dest
, base
, ofs
);
611 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
612 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
613 emit_tab(as
, PPCI_ADD
, dest
, RID_TMP
, base
);
614 emit_slwi(as
, RID_TMP
, idx
, 3);
617 /* Inlined hash lookup. Specialized for key type and for const keys.
618 ** The equivalent C code is:
619 ** Node *n = hashkey(t, key);
621 ** if (lj_obj_equal(&n->key, key)) return &n->val;
622 ** } while ((n = nextnode(n)));
625 static void asm_href(ASMState
*as
, IRIns
*ir
, IROp merge
)
627 RegSet allow
= RSET_GPR
;
628 int destused
= ra_used(ir
);
629 Reg dest
= ra_dest(as
, ir
, allow
);
630 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
631 Reg key
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
632 Reg tisnum
= RID_NONE
, tmpnum
= RID_NONE
;
633 IRRef refkey
= ir
->op2
;
634 IRIns
*irkey
= IR(refkey
);
635 IRType1 kt
= irkey
->t
;
637 MCLabel l_end
, l_loop
, l_next
;
639 rset_clear(allow
, tab
);
641 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
642 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
643 tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
644 rset_clear(allow
, tisnum
);
645 } else if (!irt_ispri(kt
)) {
646 key
= ra_alloc1(as
, refkey
, allow
);
647 rset_clear(allow
, key
);
649 tmp2
= ra_scratch(as
, allow
);
650 rset_clear(allow
, tmp2
);
652 /* Key not found in chain: jump to exit (if merged) or load niltv. */
653 l_end
= emit_label(as
);
656 asm_guardcc(as
, CC_EQ
);
658 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
660 /* Follow hash chain until the end. */
662 emit_ai(as
, PPCI_CMPWI
, dest
, 0);
663 emit_tai(as
, PPCI_LWZ
, dest
, dest
, (int32_t)offsetof(Node
, next
));
664 l_next
= emit_label(as
);
666 /* Type and value comparison. */
668 asm_guardcc(as
, CC_EQ
);
670 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
672 emit_fab(as
, PPCI_FCMPU
, 0, tmpnum
, key
);
673 emit_condbranch(as
, PPCI_BC
, CC_GE
, l_next
);
674 emit_ab(as
, PPCI_CMPLW
, tmp1
, tisnum
);
675 emit_fai(as
, PPCI_LFD
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
677 if (!irt_ispri(kt
)) {
678 emit_ab(as
, PPCI_CMPW
, tmp2
, key
);
679 emit_condbranch(as
, PPCI_BC
, CC_NE
, l_next
);
681 emit_ai(as
, PPCI_CMPWI
, tmp1
, irt_toitype(irkey
->t
));
683 emit_tai(as
, PPCI_LWZ
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
685 emit_tai(as
, PPCI_LWZ
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
686 *l_loop
= PPCI_BC
| PPCF_Y
| PPCF_CC(CC_NE
) |
687 (((char *)as
->mcp
-(char *)l_loop
) & 0xffffu
);
689 /* Load main position relative to tab->node into dest. */
690 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
692 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
695 if (irref_isk(refkey
))
696 tmphash
= ra_allock(as
, khash
, allow
);
697 emit_tab(as
, PPCI_ADD
, dest
, dest
, tmp1
);
698 emit_tai(as
, PPCI_MULLI
, tmp1
, tmp1
, sizeof(Node
));
699 emit_asb(as
, PPCI_AND
, tmp1
, tmp2
, tmphash
);
700 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
701 emit_tai(as
, PPCI_LWZ
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
702 if (irref_isk(refkey
)) {
704 } else if (irt_isstr(kt
)) {
705 emit_tai(as
, PPCI_LWZ
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
706 } else { /* Must match with hash*() in lj_tab.c. */
707 emit_tab(as
, PPCI_SUBF
, tmp1
, tmp2
, tmp1
);
708 emit_rotlwi(as
, tmp2
, tmp2
, HASH_ROT3
);
709 emit_asb(as
, PPCI_XOR
, tmp1
, tmp1
, tmp2
);
710 emit_rotlwi(as
, tmp1
, tmp1
, (HASH_ROT2
+HASH_ROT1
)&31);
711 emit_tab(as
, PPCI_SUBF
, tmp2
, dest
, tmp2
);
713 int32_t ofs
= ra_spill(as
, irkey
);
714 emit_asb(as
, PPCI_XOR
, tmp2
, tmp2
, tmp1
);
715 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
716 emit_tab(as
, PPCI_ADD
, tmp1
, tmp1
, tmp1
);
717 emit_tai(as
, PPCI_LWZ
, tmp2
, RID_SP
, ofs
+4);
718 emit_tai(as
, PPCI_LWZ
, tmp1
, RID_SP
, ofs
);
720 emit_asb(as
, PPCI_XOR
, tmp2
, key
, tmp1
);
721 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
722 emit_tai(as
, PPCI_ADDI
, tmp1
, tmp2
, HASH_BIAS
);
723 emit_tai(as
, PPCI_ADDIS
, tmp2
, key
, (HASH_BIAS
+ 32768)>>16);
729 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
731 IRIns
*kslot
= IR(ir
->op2
);
732 IRIns
*irkey
= IR(kslot
->op1
);
733 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
734 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
735 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
736 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
737 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
738 RegSet allow
= rset_exclude(RSET_GPR
, node
);
739 lua_assert(ofs
% sizeof(Node
) == 0);
742 rset_clear(allow
, dest
);
743 kofs
= (int32_t)offsetof(Node
, key
);
744 } else if (ra_hasreg(dest
)) {
745 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
747 asm_guardcc(as
, CC_NE
);
748 if (!irt_ispri(irkey
->t
)) {
749 key
= ra_scratch(as
, allow
);
750 rset_clear(allow
, key
);
752 rset_clear(allow
, type
);
753 if (irt_isnum(irkey
->t
)) {
754 emit_cmpi(as
, key
, (int32_t)ir_knum(irkey
)->u32
.lo
);
755 asm_guardcc(as
, CC_NE
);
756 emit_cmpi(as
, type
, (int32_t)ir_knum(irkey
)->u32
.hi
);
758 if (ra_hasreg(key
)) {
759 emit_cmpi(as
, key
, irkey
->i
); /* May use RID_TMP, i.e. type. */
760 asm_guardcc(as
, CC_NE
);
762 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(irkey
->t
));
764 if (ra_hasreg(key
)) emit_tai(as
, PPCI_LWZ
, key
, idx
, kofs
+4);
765 emit_tai(as
, PPCI_LWZ
, type
, idx
, kofs
);
767 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (ofs
+ 32768) >> 16);
768 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
772 static void asm_newref(ASMState
*as
, IRIns
*ir
)
774 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
776 if (ir
->r
== RID_SINK
)
778 args
[0] = ASMREF_L
; /* lua_State *L */
779 args
[1] = ir
->op1
; /* GCtab *t */
780 args
[2] = ASMREF_TMP1
; /* cTValue *key */
781 asm_setupresult(as
, ir
, ci
); /* TValue * */
782 asm_gencall(as
, ci
, args
);
783 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op2
);
786 static void asm_uref(ASMState
*as
, IRIns
*ir
)
788 /* NYI: Check that UREFO is still open and not aliasing a slot. */
789 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
790 if (irref_isk(ir
->op1
)) {
791 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
792 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
793 emit_lsptr(as
, PPCI_LWZ
, dest
, v
, RSET_GPR
);
795 Reg uv
= ra_scratch(as
, RSET_GPR
);
796 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
797 if (ir
->o
== IR_UREFC
) {
798 asm_guardcc(as
, CC_NE
);
799 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, 1);
800 emit_tai(as
, PPCI_ADDI
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
801 emit_tai(as
, PPCI_LBZ
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
803 emit_tai(as
, PPCI_LWZ
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
805 emit_tai(as
, PPCI_LWZ
, uv
, func
,
806 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
810 static void asm_fref(ASMState
*as
, IRIns
*ir
)
812 UNUSED(as
); UNUSED(ir
);
813 lua_assert(!ra_used(ir
));
816 static void asm_strref(ASMState
*as
, IRIns
*ir
)
818 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
819 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
820 int32_t ofs
= (int32_t)sizeof(GCstr
);
822 if (irref_isk(ref
)) {
823 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
824 } else if (!irref_isk(refk
)) {
825 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
826 IRIns
*irr
= IR(ir
->op2
);
827 if (ra_hasreg(irr
->r
)) {
828 ra_noweak(as
, irr
->r
);
830 } else if (mayfuse(as
, irr
->op2
) &&
831 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
832 checki16(ofs
+ IR(irr
->op2
)->i
)) {
833 ofs
+= IR(irr
->op2
)->i
;
834 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
836 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
838 emit_tai(as
, PPCI_ADDI
, dest
, dest
, ofs
);
839 emit_tab(as
, PPCI_ADD
, dest
, left
, right
);
842 r
= ra_alloc1(as
, ref
, RSET_GPR
);
845 emit_tai(as
, PPCI_ADDI
, dest
, r
, ofs
);
847 emit_tab(as
, PPCI_ADD
, dest
, r
,
848 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
851 /* -- Loads and stores ---------------------------------------------------- */
853 static PPCIns
asm_fxloadins(IRIns
*ir
)
855 switch (irt_type(ir
->t
)) {
856 case IRT_I8
: return PPCI_LBZ
; /* Needs sign-extension. */
857 case IRT_U8
: return PPCI_LBZ
;
858 case IRT_I16
: return PPCI_LHA
;
859 case IRT_U16
: return PPCI_LHZ
;
860 case IRT_NUM
: return PPCI_LFD
;
861 case IRT_FLOAT
: return PPCI_LFS
;
862 default: return PPCI_LWZ
;
866 static PPCIns
asm_fxstoreins(IRIns
*ir
)
868 switch (irt_type(ir
->t
)) {
869 case IRT_I8
: case IRT_U8
: return PPCI_STB
;
870 case IRT_I16
: case IRT_U16
: return PPCI_STH
;
871 case IRT_NUM
: return PPCI_STFD
;
872 case IRT_FLOAT
: return PPCI_STFS
;
873 default: return PPCI_STW
;
877 static void asm_fload(ASMState
*as
, IRIns
*ir
)
879 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
880 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
881 PPCIns pi
= asm_fxloadins(ir
);
883 if (ir
->op2
== IRFL_TAB_ARRAY
) {
884 ofs
= asm_fuseabase(as
, ir
->op1
);
885 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
886 emit_tai(as
, PPCI_ADDI
, dest
, idx
, ofs
);
890 ofs
= field_ofs
[ir
->op2
];
891 lua_assert(!irt_isi8(ir
->t
));
892 emit_tai(as
, pi
, dest
, idx
, ofs
);
895 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
897 if (ir
->r
!= RID_SINK
) {
898 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
899 IRIns
*irf
= IR(ir
->op1
);
900 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
901 int32_t ofs
= field_ofs
[irf
->op2
];
902 PPCIns pi
= asm_fxstoreins(ir
);
903 emit_tai(as
, pi
, src
, idx
, ofs
);
907 static void asm_xload(ASMState
*as
, IRIns
*ir
)
909 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
910 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
912 emit_as(as
, PPCI_EXTSB
, dest
, dest
);
913 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
916 static void asm_xstore(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
919 if (ir
->r
== RID_SINK
)
921 if (ofs
== 0 && mayfuse(as
, ir
->op2
) && (irb
= IR(ir
->op2
))->o
== IR_BSWAP
&&
922 ra_noreg(irb
->r
) && (irt_isint(ir
->t
) || irt_isu32(ir
->t
))) {
923 /* Fuse BSWAP with XSTORE to stwbrx. */
924 Reg src
= ra_alloc1(as
, irb
->op1
, RSET_GPR
);
925 asm_fusexrefx(as
, PPCI_STWBRX
, src
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
927 Reg src
= ra_alloc1(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
928 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
929 rset_exclude(RSET_GPR
, src
), ofs
);
933 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
936 Reg dest
= RID_NONE
, type
= RID_TMP
, tmp
= RID_TMP
, idx
;
937 RegSet allow
= RSET_GPR
;
938 int32_t ofs
= AHUREF_LSX
;
940 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
941 if (!irt_isnum(t
)) ofs
= 0;
942 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
943 rset_clear(allow
, dest
);
945 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
947 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, rset_exclude(allow
, idx
));
948 asm_guardcc(as
, CC_GE
);
949 emit_ab(as
, PPCI_CMPLW
, type
, tisnum
);
950 if (ra_hasreg(dest
)) {
951 if (ofs
== AHUREF_LSX
) {
952 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
,
953 (idx
&255)), (idx
>>8)));
954 emit_fab(as
, PPCI_LFDX
, dest
, (idx
&255), tmp
);
956 emit_fai(as
, PPCI_LFD
, dest
, idx
, ofs
);
960 asm_guardcc(as
, CC_NE
);
961 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(t
));
962 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, idx
, ofs
+4);
964 if (ofs
== AHUREF_LSX
) {
965 emit_tab(as
, PPCI_LWZX
, type
, (idx
&255), tmp
);
966 emit_slwi(as
, tmp
, (idx
>>8), 3);
968 emit_tai(as
, PPCI_LWZ
, type
, idx
, ofs
);
972 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
974 RegSet allow
= RSET_GPR
;
975 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
976 int32_t ofs
= AHUREF_LSX
;
977 if (ir
->r
== RID_SINK
)
979 if (irt_isnum(ir
->t
)) {
980 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
982 if (!irt_ispri(ir
->t
)) {
983 src
= ra_alloc1(as
, ir
->op2
, allow
);
984 rset_clear(allow
, src
);
987 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
988 rset_clear(allow
, type
);
990 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
991 if (irt_isnum(ir
->t
)) {
992 if (ofs
== AHUREF_LSX
) {
993 emit_fab(as
, PPCI_STFDX
, src
, (idx
&255), RID_TMP
);
994 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
996 emit_fai(as
, PPCI_STFD
, src
, idx
, ofs
);
1000 emit_tai(as
, PPCI_STW
, src
, idx
, ofs
+4);
1001 if (ofs
== AHUREF_LSX
) {
1002 emit_tab(as
, PPCI_STWX
, type
, (idx
&255), RID_TMP
);
1003 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
1005 emit_tai(as
, PPCI_STW
, type
, idx
, ofs
);
1010 static void asm_sload(ASMState
*as
, IRIns
*ir
)
1012 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 0 : 4);
1014 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
1015 RegSet allow
= RSET_GPR
;
1016 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
1017 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
1018 lua_assert(LJ_DUALNUM
||
1019 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
1020 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
1021 dest
= ra_scratch(as
, RSET_FPR
);
1022 asm_tointg(as
, ir
, dest
);
1023 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
1024 } else if (ra_used(ir
)) {
1025 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
1026 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
1027 rset_clear(allow
, dest
);
1028 base
= ra_alloc1(as
, REF_BASE
, allow
);
1029 rset_clear(allow
, base
);
1030 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1032 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
1033 dest
= ra_scratch(as
, RSET_FPR
);
1034 emit_fai(as
, PPCI_STFD
, dest
, RID_SP
, SPOFS_TMP
);
1035 emit_fb(as
, PPCI_FCTIWZ
, dest
, dest
);
1036 t
.irt
= IRT_NUM
; /* Check for original type. */
1038 Reg tmp
= ra_scratch(as
, allow
);
1039 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, tmp
));
1040 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
1041 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
1042 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
1043 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
1044 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
1045 rset_clear(allow
, hibias
));
1046 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPLO
);
1047 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
1048 emit_asi(as
, PPCI_XORIS
, tmp
, tmp
, 0x8000);
1050 t
.irt
= IRT_INT
; /* Check for original type. */
1055 base
= ra_alloc1(as
, REF_BASE
, allow
);
1056 rset_clear(allow
, base
);
1059 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1060 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
1061 asm_guardcc(as
, CC_GE
);
1062 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tisnum
);
1065 if (ra_hasreg(dest
)) emit_fai(as
, PPCI_LFD
, dest
, base
, ofs
-4);
1067 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1068 asm_guardcc(as
, CC_NE
);
1069 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, irt_toitype(t
));
1072 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, base
, ofs
);
1074 if (ra_hasreg(type
)) emit_tai(as
, PPCI_LWZ
, type
, base
, ofs
-4);
1077 /* -- Allocations --------------------------------------------------------- */
1080 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1082 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1083 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1084 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1085 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1086 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1088 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1089 RegSet drop
= RSET_SCRATCH
;
1090 lua_assert(sz
!= CTSIZE_INVALID
);
1092 args
[0] = ASMREF_L
; /* lua_State *L */
1093 args
[1] = ASMREF_TMP1
; /* MSize size */
1096 if (ra_hasreg(ir
->r
))
1097 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1098 ra_evictset(as
, drop
);
1100 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1102 /* Initialize immutable cdata object. */
1103 if (ir
->o
== IR_CNEWI
) {
1104 int32_t ofs
= sizeof(GCcdata
);
1105 lua_assert(sz
== 4 || sz
== 8);
1108 lua_assert((ir
+1)->o
== IR_HIOP
);
1111 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1112 emit_tai(as
, PPCI_STW
, r
, RID_RET
, ofs
);
1113 rset_clear(allow
, r
);
1114 if (ofs
== sizeof(GCcdata
)) break;
1118 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1119 emit_tai(as
, PPCI_STB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1120 emit_tai(as
, PPCI_STH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1121 emit_ti(as
, PPCI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1122 emit_ti(as
, PPCI_LI
, RID_TMP
, ctypeid
); /* Lower 16 bit used. Sign-ext ok. */
1123 asm_gencall(as
, ci
, args
);
1124 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1125 ra_releasetmp(as
, ASMREF_TMP1
));
1128 #define asm_cnew(as, ir) ((void)0)
1131 /* -- Write barriers ------------------------------------------------------ */
1133 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1135 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1136 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1138 MCLabel l_end
= emit_label(as
);
1139 emit_tai(as
, PPCI_STW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1140 emit_tai(as
, PPCI_STB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1141 emit_setgl(as
, tab
, gc
.grayagain
);
1142 lua_assert(LJ_GC_BLACK
== 0x04);
1143 emit_rot(as
, PPCI_RLWINM
, mark
, mark
, 0, 30, 28); /* Clear black bit. */
1144 emit_getgl(as
, link
, gc
.grayagain
);
1145 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1146 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, mark
, LJ_GC_BLACK
);
1147 emit_tai(as
, PPCI_LBZ
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1150 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1152 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1156 /* No need for other object barriers (yet). */
1157 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1158 ra_evictset(as
, RSET_SCRATCH
);
1159 l_end
= emit_label(as
);
1160 args
[0] = ASMREF_TMP1
; /* global_State *g */
1161 args
[1] = ir
->op1
; /* TValue *tv */
1162 asm_gencall(as
, ci
, args
);
1163 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1164 obj
= IR(ir
->op1
)->r
;
1165 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1166 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1167 emit_asi(as
, PPCI_ANDIDOT
, tmp
, tmp
, LJ_GC_BLACK
);
1168 emit_condbranch(as
, PPCI_BC
, CC_EQ
, l_end
);
1169 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1170 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1171 emit_tai(as
, PPCI_LBZ
, tmp
, obj
,
1172 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1173 emit_tai(as
, PPCI_LBZ
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1176 /* -- Arithmetic and logic operations ------------------------------------- */
1178 static void asm_fparith(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1180 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1181 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1182 right
= (left
>> 8); left
&= 255;
1183 if (pi
== PPCI_FMUL
)
1184 emit_fac(as
, pi
, dest
, left
, right
);
1186 emit_fab(as
, pi
, dest
, left
, right
);
1189 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1191 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1192 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1193 emit_fb(as
, pi
, dest
, left
);
1196 static int asm_fpjoin_pow(ASMState
*as
, IRIns
*ir
)
1198 IRIns
*irp
= IR(ir
->op1
);
1199 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1200 IRIns
*irpp
= IR(irp
->op1
);
1201 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1202 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1203 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_pow
];
1205 args
[0] = irpp
->op1
;
1207 asm_setupresult(as
, ir
, ci
);
1208 asm_gencall(as
, ci
, args
);
1215 static void asm_add(ASMState
*as
, IRIns
*ir
)
1217 if (irt_isnum(ir
->t
)) {
1218 if (!asm_fusemadd(as
, ir
, PPCI_FMADD
, PPCI_FMADD
))
1219 asm_fparith(as
, ir
, PPCI_FADD
);
1221 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1222 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1224 if (irref_isk(ir
->op2
)) {
1225 int32_t k
= IR(ir
->op2
)->i
;
1228 /* May fail due to spills/restores above, but simplifies the logic. */
1229 if (as
->flagmcp
== as
->mcp
) {
1234 emit_tai(as
, pi
, dest
, left
, k
);
1236 } else if ((k
& 0xffff) == 0) {
1237 emit_tai(as
, PPCI_ADDIS
, dest
, left
, (k
>> 16));
1239 } else if (!as
->sectref
) {
1240 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (k
+ 32768) >> 16);
1241 emit_tai(as
, PPCI_ADDI
, dest
, left
, k
);
1246 /* May fail due to spills/restores above, but simplifies the logic. */
1247 if (as
->flagmcp
== as
->mcp
) {
1252 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1253 emit_tab(as
, pi
, dest
, left
, right
);
1257 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1259 if (irt_isnum(ir
->t
)) {
1260 if (!asm_fusemadd(as
, ir
, PPCI_FMSUB
, PPCI_FNMSUB
))
1261 asm_fparith(as
, ir
, PPCI_FSUB
);
1263 PPCIns pi
= PPCI_SUBF
;
1264 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1266 if (irref_isk(ir
->op1
)) {
1267 int32_t k
= IR(ir
->op1
)->i
;
1269 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1270 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1274 /* May fail due to spills/restores above, but simplifies the logic. */
1275 if (as
->flagmcp
== as
->mcp
) {
1280 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1281 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1282 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1286 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1288 if (irt_isnum(ir
->t
)) {
1289 asm_fparith(as
, ir
, PPCI_FMUL
);
1291 PPCIns pi
= PPCI_MULLW
;
1292 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1293 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1294 if (irref_isk(ir
->op2
)) {
1295 int32_t k
= IR(ir
->op2
)->i
;
1297 emit_tai(as
, PPCI_MULLI
, dest
, left
, k
);
1301 /* May fail due to spills/restores above, but simplifies the logic. */
1302 if (as
->flagmcp
== as
->mcp
) {
1307 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1308 emit_tab(as
, pi
, dest
, left
, right
);
1312 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1314 if (irt_isnum(ir
->t
)) {
1315 asm_fpunary(as
, ir
, PPCI_FNEG
);
1318 PPCIns pi
= PPCI_NEG
;
1319 if (as
->flagmcp
== as
->mcp
) {
1324 dest
= ra_dest(as
, ir
, RSET_GPR
);
1325 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1326 emit_tab(as
, pi
, dest
, left
, 0);
1330 static void asm_arithov(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1332 Reg dest
, left
, right
;
1333 if (as
->flagmcp
== as
->mcp
) {
1337 asm_guardcc(as
, CC_SO
);
1338 dest
= ra_dest(as
, ir
, RSET_GPR
);
1339 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1340 right
= (left
>> 8); left
&= 255;
1341 if (pi
== PPCI_SUBFO
) { Reg tmp
= left
; left
= right
; right
= tmp
; }
1342 emit_tab(as
, pi
|PPCF_DOT
, dest
, left
, right
);
1346 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1348 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1349 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1350 PPCIns pi
= PPCI_ADDE
;
1351 if (irref_isk(ir
->op2
)) {
1352 int32_t k
= IR(ir
->op2
)->i
;
1362 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1364 emit_tab(as
, pi
, dest
, left
, right
);
1366 dest
= ra_dest(as
, ir
, RSET_GPR
);
1367 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1368 if (irref_isk(ir
->op2
)) {
1369 int32_t k
= IR(ir
->op2
)->i
;
1371 emit_tai(as
, PPCI_ADDIC
, dest
, left
, k
);
1375 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1376 emit_tab(as
, PPCI_ADDC
, dest
, left
, right
);
1379 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1381 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1382 Reg left
, right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1383 PPCIns pi
= PPCI_SUBFE
;
1384 if (irref_isk(ir
->op1
)) {
1385 int32_t k
= IR(ir
->op1
)->i
;
1395 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1397 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1399 dest
= ra_dest(as
, ir
, RSET_GPR
);
1400 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1401 if (irref_isk(ir
->op1
)) {
1402 int32_t k
= IR(ir
->op1
)->i
;
1404 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1408 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1409 emit_tab(as
, PPCI_SUBFC
, dest
, right
, left
);
1412 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1414 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1415 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1416 emit_tab(as
, PPCI_SUBFZE
, dest
, left
, 0);
1418 dest
= ra_dest(as
, ir
, RSET_GPR
);
1419 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1420 emit_tai(as
, PPCI_SUBFIC
, dest
, left
, 0);
1424 static void asm_bitnot(ASMState
*as
, IRIns
*ir
)
1426 Reg dest
, left
, right
;
1427 PPCIns pi
= PPCI_NOR
;
1428 if (as
->flagmcp
== as
->mcp
) {
1433 dest
= ra_dest(as
, ir
, RSET_GPR
);
1434 if (mayfuse(as
, ir
->op1
)) {
1435 IRIns
*irl
= IR(ir
->op1
);
1436 if (irl
->o
== IR_BAND
)
1437 pi
^= (PPCI_NOR
^ PPCI_NAND
);
1438 else if (irl
->o
== IR_BXOR
)
1439 pi
^= (PPCI_NOR
^ PPCI_EQV
);
1440 else if (irl
->o
!= IR_BOR
)
1442 left
= ra_hintalloc(as
, irl
->op1
, dest
, RSET_GPR
);
1443 right
= ra_alloc1(as
, irl
->op2
, rset_exclude(RSET_GPR
, left
));
1446 left
= right
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1448 emit_asb(as
, pi
, dest
, left
, right
);
1451 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1453 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1455 if (mayfuse(as
, ir
->op1
) && (irx
= IR(ir
->op1
))->o
== IR_XLOAD
&&
1456 ra_noreg(irx
->r
) && (irt_isint(irx
->t
) || irt_isu32(irx
->t
))) {
1457 /* Fuse BSWAP with XLOAD to lwbrx. */
1458 asm_fusexrefx(as
, PPCI_LWBRX
, dest
, irx
->op1
, RSET_GPR
);
1460 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1464 emit_mr(as
, dest
, RID_TMP
);
1466 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 16, 23);
1467 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 0, 7);
1468 emit_rotlwi(as
, tmp
, left
, 8);
1472 static void asm_bitop(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1474 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1475 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1476 if (irref_isk(ir
->op2
)) {
1477 int32_t k
= IR(ir
->op2
)->i
;
1479 if ((checku16(k
) || (k
& 0xffff) == 0) || (tmp
= dest
, !as
->sectref
)) {
1481 emit_asi(as
, pik
^ (PPCI_ORI
^ PPCI_ORIS
), dest
, tmp
, (k
>> 16));
1482 if ((k
& 0xffff) == 0) return;
1484 emit_asi(as
, pik
, dest
, left
, k
);
1488 /* May fail due to spills/restores above, but simplifies the logic. */
1489 if (as
->flagmcp
== as
->mcp
) {
1494 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1495 emit_asb(as
, pi
, dest
, left
, right
);
1498 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1499 static void asm_fuseandsh(ASMState
*as
, PPCIns pi
, int32_t mask
, IRRef ref
)
1503 if (mayfuse(as
, ref
) && (ir
= IR(ref
), ra_noreg(ir
->r
)) &&
1504 irref_isk(ir
->op2
) && ir
->o
>= IR_BSHL
&& ir
->o
<= IR_BROR
) {
1505 int32_t sh
= (IR(ir
->op2
)->i
& 31);
1508 if ((mask
& ((1u<<sh
)-1))) goto nofuse
;
1511 if ((mask
& ~((~0u)>>sh
))) goto nofuse
;
1519 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1520 *--as
->mcp
= pi
| PPCF_T(left
) | PPCF_B(sh
);
1524 left
= ra_alloc1(as
, ref
, RSET_GPR
);
1525 *--as
->mcp
= pi
| PPCF_T(left
);
1528 static void asm_bitand(ASMState
*as
, IRIns
*ir
)
1530 Reg dest
, left
, right
;
1531 IRRef lref
= ir
->op1
;
1534 if (as
->flagmcp
== as
->mcp
) {
1539 dest
= ra_dest(as
, ir
, RSET_GPR
);
1540 if (irref_isk(ir
->op2
)) {
1541 int32_t k
= IR(ir
->op2
)->i
;
1543 /* First check for a contiguous bitmask as used by rlwinm. */
1544 uint32_t s1
= lj_ffs((uint32_t)k
);
1545 uint32_t k1
= ((uint32_t)k
>> s1
);
1546 if ((k1
& (k1
+1)) == 0) {
1547 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1548 PPCF_MB(31-lj_fls((uint32_t)k
)) | PPCF_ME(31-s1
),
1553 uint32_t s2
= lj_ffs(~(uint32_t)k
);
1554 uint32_t k2
= (~(uint32_t)k
>> s2
);
1555 if ((k2
& (k2
+1)) == 0) {
1556 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1557 PPCF_MB(32-s2
) | PPCF_ME(30-lj_fls(~(uint32_t)k
)),
1564 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1565 emit_asi(as
, PPCI_ANDIDOT
, dest
, left
, k
);
1567 } else if ((k
& 0xffff) == 0) {
1568 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1569 emit_asi(as
, PPCI_ANDISDOT
, dest
, left
, (k
>> 16));
1574 if (mayfuse(as
, op2
) && IR(op2
)->o
== IR_BNOT
&& ra_noreg(IR(op2
)->r
)) {
1575 dot
^= (PPCI_AND
^ PPCI_ANDC
);
1578 left
= ra_hintalloc(as
, lref
, dest
, RSET_GPR
);
1579 right
= ra_alloc1(as
, op2
, rset_exclude(RSET_GPR
, left
));
1580 emit_asb(as
, PPCI_AND
^ dot
, dest
, left
, right
);
1583 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1587 if (as
->flagmcp
== as
->mcp
) {
1592 dest
= ra_dest(as
, ir
, RSET_GPR
);
1593 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1594 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1595 int32_t shift
= (IR(ir
->op2
)->i
& 31);
1596 if (pik
== 0) /* SLWI */
1597 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, shift
, 0, 31-shift
);
1598 else if (pik
== 1) /* SRWI */
1599 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, (32-shift
)&31, shift
, 31);
1601 emit_asb(as
, pik
|dot
, dest
, left
, shift
);
1603 Reg right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1604 emit_asb(as
, pi
|dot
, dest
, left
, right
);
1608 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1610 if (irt_isnum(ir
->t
)) {
1611 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1613 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1614 right
= (left
>> 8); left
&= 255;
1615 if (tmp
== left
|| tmp
== right
)
1616 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_FPR
,
1617 dest
), left
), right
));
1618 emit_facb(as
, PPCI_FSEL
, dest
, tmp
,
1619 ismax
? left
: right
, ismax
? right
: left
);
1620 emit_fab(as
, PPCI_FSUB
, tmp
, left
, right
);
1622 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1623 Reg tmp1
= RID_TMP
, tmp2
= dest
;
1624 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1625 right
= (left
>> 8); left
&= 255;
1626 if (tmp2
== left
|| tmp2
== right
)
1627 tmp2
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
,
1628 dest
), left
), right
));
1629 emit_tab(as
, PPCI_ADD
, dest
, tmp2
, right
);
1630 emit_asb(as
, ismax
? PPCI_ANDC
: PPCI_AND
, tmp2
, tmp2
, tmp1
);
1631 emit_tab(as
, PPCI_SUBFE
, tmp1
, tmp1
, tmp1
);
1632 emit_tab(as
, PPCI_SUBFC
, tmp2
, tmp2
, tmp1
);
1633 emit_asi(as
, PPCI_XORIS
, tmp2
, right
, 0x8000);
1634 emit_asi(as
, PPCI_XORIS
, tmp1
, left
, 0x8000);
1638 /* -- Comparisons --------------------------------------------------------- */
1640 #define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
1641 #define CC_TWO 0x80 /* Check two flags for FP comparison. */
1643 /* Map of comparisons to flags. ORDER IR. */
1644 static const uint8_t asm_compmap
[IR_ABC
+1] = {
1645 /* op int cc FP cc */
1646 /* LT */ CC_GE
+ (CC_GE
<<4),
1647 /* GE */ CC_LT
+ (CC_LE
<<4) + CC_TWO
,
1648 /* LE */ CC_GT
+ (CC_GE
<<4) + CC_TWO
,
1649 /* GT */ CC_LE
+ (CC_LE
<<4),
1650 /* ULT */ CC_GE
+ CC_UNSIGNED
+ (CC_GT
<<4) + CC_TWO
,
1651 /* UGE */ CC_LT
+ CC_UNSIGNED
+ (CC_LT
<<4),
1652 /* ULE */ CC_GT
+ CC_UNSIGNED
+ (CC_GT
<<4),
1653 /* UGT */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
,
1654 /* EQ */ CC_NE
+ (CC_NE
<<4),
1655 /* NE */ CC_EQ
+ (CC_EQ
<<4),
1656 /* ABC */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
/* Same as UGT. */
1659 static void asm_intcomp_(ASMState
*as
, IRRef lref
, IRRef rref
, Reg cr
, PPCCC cc
)
1661 Reg right
, left
= ra_alloc1(as
, lref
, RSET_GPR
);
1662 if (irref_isk(rref
)) {
1663 int32_t k
= IR(rref
)->i
;
1664 if ((cc
& CC_UNSIGNED
) == 0) { /* Signed comparison with constant. */
1666 emit_tai(as
, PPCI_CMPWI
, cr
, left
, k
);
1667 /* Signed comparison with zero and referencing previous ins? */
1668 if (k
== 0 && lref
== as
->curins
-1)
1669 as
->flagmcp
= as
->mcp
; /* Allow elimination of the compare. */
1671 } else if ((cc
& 3) == (CC_EQ
& 3)) { /* Use CMPLWI for EQ or NE. */
1673 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1675 } else if (!as
->sectref
&& ra_noreg(IR(rref
)->r
)) {
1676 emit_tai(as
, PPCI_CMPLWI
, cr
, RID_TMP
, k
);
1677 emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, (k
>> 16));
1681 } else { /* Unsigned comparison with constant. */
1683 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1688 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, left
));
1689 emit_tab(as
, (cc
& CC_UNSIGNED
) ? PPCI_CMPLW
: PPCI_CMPW
, cr
, left
, right
);
1692 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1694 PPCCC cc
= asm_compmap
[ir
->o
];
1695 if (irt_isnum(ir
->t
)) {
1696 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1697 right
= (left
>> 8); left
&= 255;
1698 asm_guardcc(as
, (cc
>> 4));
1700 emit_tab(as
, PPCI_CROR
, ((cc
>>4)&3), ((cc
>>4)&3), (CC_EQ
&3));
1701 emit_fab(as
, PPCI_FCMPU
, 0, left
, right
);
1703 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1704 if (irref_isk(lref
) && !irref_isk(rref
)) {
1705 /* Swap constants to the right (only for ABC). */
1706 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1707 if ((cc
& 2) == 0) cc
^= 1; /* LT <-> GT, LE <-> GE */
1709 asm_guardcc(as
, cc
);
1710 asm_intcomp_(as
, lref
, rref
, 0, cc
);
1715 /* 64 bit integer comparisons. */
1716 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1718 PPCCC cc
= asm_compmap
[(ir
-1)->o
];
1719 if ((cc
&3) == (CC_EQ
&3)) {
1720 asm_guardcc(as
, cc
);
1721 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CROR
,
1722 (CC_EQ
&3), (CC_EQ
&3), 4+(CC_EQ
&3));
1724 asm_guardcc(as
, CC_EQ
);
1725 emit_tab(as
, PPCI_CROR
, (CC_EQ
&3), (CC_EQ
&3), ((cc
^~(cc
>>2))&1));
1726 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CRANDC
,
1727 (CC_EQ
&3), (CC_EQ
&3), 4+(cc
&3));
1729 /* Loword comparison sets cr1 and is unsigned, except for equality. */
1730 asm_intcomp_(as
, (ir
-1)->op1
, (ir
-1)->op2
, 4,
1731 cc
| ((cc
&3) == (CC_EQ
&3) ? 0 : CC_UNSIGNED
));
1732 /* Hiword comparison sets cr0. */
1733 asm_intcomp_(as
, ir
->op1
, ir
->op2
, 0, cc
);
1734 as
->flagmcp
= NULL
; /* Doesn't work here. */
1738 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1740 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1741 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1744 /* HIOP is marked as a store because it needs its own DCE logic. */
1745 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1746 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1747 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1748 as
->curins
--; /* Always skip the CONV. */
1752 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1753 as
->curins
--; /* Always skip the loword comparison. */
1756 } else if ((ir
-1)->o
== IR_XSTORE
) {
1757 as
->curins
--; /* Handle both stores here. */
1758 if ((ir
-1)->r
!= RID_SINK
) {
1759 asm_xstore(as
, ir
, 0);
1760 asm_xstore(as
, ir
-1, 4);
1764 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1765 switch ((ir
-1)->o
) {
1766 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1767 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1768 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1772 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1775 /* Nothing to do here. Handled by lo op itself. */
1777 default: lua_assert(0); break;
1780 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1784 /* -- Stack handling ------------------------------------------------------ */
1786 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1787 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1788 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1790 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1791 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1792 rset_clear(allow
, pbase
);
1793 tmp
= allow
? rset_pickbot(allow
) :
1794 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1795 emit_condbranch(as
, PPCI_BC
, CC_LT
, asm_exitstub_addr(as
, exitno
));
1796 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1797 emit_tai(as
, PPCI_LWZ
, tmp
, RID_SP
, SPOFS_TMPW
);
1799 ra_modified(as
, tmp
);
1800 emit_ai(as
, PPCI_CMPLWI
, RID_TMP
, (int32_t)(8*topslot
));
1801 emit_tab(as
, PPCI_SUBF
, RID_TMP
, pbase
, tmp
);
1802 emit_tai(as
, PPCI_LWZ
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1803 if (pbase
== RID_TMP
)
1804 emit_getgl(as
, RID_TMP
, jit_base
);
1805 emit_getgl(as
, tmp
, jit_L
);
1806 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1807 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPW
);
1810 /* Restore Lua stack from on-trace state. */
1811 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1813 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1814 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1815 MSize n
, nent
= snap
->nent
;
1816 /* Store the value of all modified slots to the Lua stack. */
1817 for (n
= 0; n
< nent
; n
++) {
1818 SnapEntry sn
= map
[n
];
1819 BCReg s
= snap_slot(sn
);
1820 int32_t ofs
= 8*((int32_t)s
-1);
1821 IRRef ref
= snap_ref(sn
);
1822 IRIns
*ir
= IR(ref
);
1823 if ((sn
& SNAP_NORESTORE
))
1825 if (irt_isnum(ir
->t
)) {
1826 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1827 emit_fai(as
, PPCI_STFD
, src
, RID_BASE
, ofs
);
1830 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1831 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1832 if (!irt_ispri(ir
->t
)) {
1833 Reg src
= ra_alloc1(as
, ref
, allow
);
1834 rset_clear(allow
, src
);
1835 emit_tai(as
, PPCI_STW
, src
, RID_BASE
, ofs
+4);
1837 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1838 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1839 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1841 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1843 emit_tai(as
, PPCI_STW
, type
, RID_BASE
, ofs
);
1847 lua_assert(map
+ nent
== flinks
);
1850 /* -- GC handling --------------------------------------------------------- */
1852 /* Check GC threshold and do one or more GC steps. */
1853 static void asm_gc_check(ASMState
*as
)
1855 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1859 ra_evictset(as
, RSET_SCRATCH
);
1860 l_end
= emit_label(as
);
1861 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1862 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
1863 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0);
1864 args
[0] = ASMREF_TMP1
; /* global_State *g */
1865 args
[1] = ASMREF_TMP2
; /* MSize steps */
1866 asm_gencall(as
, ci
, args
);
1867 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1868 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1869 emit_loadi(as
, tmp
, as
->gcsteps
);
1870 /* Jump around GC step if GC total < GC threshold. */
1871 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_LT
, l_end
);
1872 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tmp
);
1873 emit_getgl(as
, tmp
, gc
.threshold
);
1874 emit_getgl(as
, RID_TMP
, gc
.total
);
1879 /* -- Loop handling ------------------------------------------------------- */
1881 /* Fixup the loop branch. */
1882 static void asm_loop_fixup(ASMState
*as
)
1884 MCode
*p
= as
->mctop
;
1885 MCode
*target
= as
->mcp
;
1886 if (as
->loopinv
) { /* Inverted loop branch? */
1887 /* asm_guardcc already inverted the cond branch and patched the final b. */
1888 p
[-2] = (p
[-2] & (0xffff0000u
& ~PPCF_Y
)) | (((target
-p
+2) & 0x3fffu
) << 2);
1890 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1894 /* -- Head of trace ------------------------------------------------------- */
1896 /* Coalesce BASE register for a root trace. */
1897 static void asm_head_root_base(ASMState
*as
)
1899 IRIns
*ir
= IR(REF_BASE
);
1903 if (rset_test(as
->modset
, r
))
1904 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1906 emit_mr(as
, r
, RID_BASE
);
1910 /* Coalesce BASE register for a side trace. */
1911 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1913 IRIns
*ir
= IR(REF_BASE
);
1917 if (rset_test(as
->modset
, r
))
1918 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1920 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1921 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1922 rset_clear(allow
, irp
->r
);
1923 emit_mr(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1925 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1931 /* -- Tail of trace ------------------------------------------------------- */
1933 /* Fixup the tail code. */
1934 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1936 MCode
*p
= as
->mctop
;
1938 int32_t spadj
= as
->T
->spadjust
;
1944 /* Patch stack adjustment. */
1945 lua_assert(checki16(CFRAME_SIZE
+spadj
));
1946 p
[-3] = PPCI_ADDI
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | (CFRAME_SIZE
+spadj
);
1947 p
[-2] = PPCI_STWU
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | spadj
;
1949 /* Patch exit branch. */
1950 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1951 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1954 /* Prepare tail of code. */
1955 static void asm_tail_prep(ASMState
*as
)
1957 MCode
*p
= as
->mctop
- 1; /* Leave room for exit branch. */
1959 as
->invmcp
= as
->mcp
= p
;
1961 as
->mcp
= p
-2; /* Leave room for stack pointer adjustment. */
1966 /* -- Instruction dispatch ------------------------------------------------ */
1968 /* Assemble a single instruction. */
1969 static void asm_ir(ASMState
*as
, IRIns
*ir
)
1971 switch ((IROp
)ir
->o
) {
1972 /* Miscellaneous ops. */
1973 case IR_LOOP
: asm_loop(as
); break;
1974 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
1976 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
1977 case IR_PHI
: asm_phi(as
, ir
); break;
1978 case IR_HIOP
: asm_hiop(as
, ir
); break;
1979 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
1981 /* Guarded assertions. */
1982 case IR_EQ
: case IR_NE
:
1983 if ((ir
-1)->o
== IR_HREF
&& ir
->op1
== as
->curins
-1) {
1985 asm_href(as
, ir
-1, (IROp
)ir
->o
);
1989 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
1990 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
1995 case IR_RETF
: asm_retf(as
, ir
); break;
1998 case IR_BNOT
: asm_bitnot(as
, ir
); break;
1999 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
2001 case IR_BAND
: asm_bitand(as
, ir
); break;
2002 case IR_BOR
: asm_bitop(as
, ir
, PPCI_OR
, PPCI_ORI
); break;
2003 case IR_BXOR
: asm_bitop(as
, ir
, PPCI_XOR
, PPCI_XORI
); break;
2005 case IR_BSHL
: asm_bitshift(as
, ir
, PPCI_SLW
, 0); break;
2006 case IR_BSHR
: asm_bitshift(as
, ir
, PPCI_SRW
, 1); break;
2007 case IR_BSAR
: asm_bitshift(as
, ir
, PPCI_SRAW
, PPCI_SRAWI
); break;
2008 case IR_BROL
: asm_bitshift(as
, ir
, PPCI_RLWNM
|PPCF_MB(0)|PPCF_ME(31),
2009 PPCI_RLWINM
|PPCF_MB(0)|PPCF_ME(31)); break;
2010 case IR_BROR
: lua_assert(0); break;
2012 /* Arithmetic ops. */
2013 case IR_ADD
: asm_add(as
, ir
); break;
2014 case IR_SUB
: asm_sub(as
, ir
); break;
2015 case IR_MUL
: asm_mul(as
, ir
); break;
2016 case IR_DIV
: asm_fparith(as
, ir
, PPCI_FDIV
); break;
2017 case IR_MOD
: asm_callid(as
, ir
, IRCALL_lj_vm_modi
); break;
2018 case IR_POW
: asm_callid(as
, ir
, IRCALL_lj_vm_powi
); break;
2019 case IR_NEG
: asm_neg(as
, ir
); break;
2021 case IR_ABS
: asm_fpunary(as
, ir
, PPCI_FABS
); break;
2022 case IR_ATAN2
: asm_callid(as
, ir
, IRCALL_atan2
); break;
2023 case IR_LDEXP
: asm_callid(as
, ir
, IRCALL_ldexp
); break;
2024 case IR_MIN
: asm_min_max(as
, ir
, 0); break;
2025 case IR_MAX
: asm_min_max(as
, ir
, 1); break;
2027 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
2029 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
2032 /* Overflow-checking arithmetic ops. */
2033 case IR_ADDOV
: asm_arithov(as
, ir
, PPCI_ADDO
); break;
2034 case IR_SUBOV
: asm_arithov(as
, ir
, PPCI_SUBFO
); break;
2035 case IR_MULOV
: asm_arithov(as
, ir
, PPCI_MULLWO
); break;
2037 /* Memory references. */
2038 case IR_AREF
: asm_aref(as
, ir
); break;
2039 case IR_HREF
: asm_href(as
, ir
, 0); break;
2040 case IR_HREFK
: asm_hrefk(as
, ir
); break;
2041 case IR_NEWREF
: asm_newref(as
, ir
); break;
2042 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
2043 case IR_FREF
: asm_fref(as
, ir
); break;
2044 case IR_STRREF
: asm_strref(as
, ir
); break;
2046 /* Loads and stores. */
2047 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
2048 asm_ahuvload(as
, ir
);
2050 case IR_FLOAD
: asm_fload(as
, ir
); break;
2051 case IR_XLOAD
: asm_xload(as
, ir
); break;
2052 case IR_SLOAD
: asm_sload(as
, ir
); break;
2054 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
2055 case IR_FSTORE
: asm_fstore(as
, ir
); break;
2056 case IR_XSTORE
: asm_xstore(as
, ir
, 0); break;
2059 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
2060 case IR_TNEW
: asm_tnew(as
, ir
); break;
2061 case IR_TDUP
: asm_tdup(as
, ir
); break;
2062 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
2064 /* Write barriers. */
2065 case IR_TBAR
: asm_tbar(as
, ir
); break;
2066 case IR_OBAR
: asm_obar(as
, ir
); break;
2068 /* Type conversions. */
2069 case IR_CONV
: asm_conv(as
, ir
); break;
2070 case IR_TOBIT
: asm_tobit(as
, ir
); break;
2071 case IR_TOSTR
: asm_tostr(as
, ir
); break;
2072 case IR_STRTO
: asm_strto(as
, ir
); break;
2075 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
2076 case IR_CALLXS
: asm_callx(as
, ir
); break;
2077 case IR_CARG
: break;
2080 setintV(&as
->J
->errinfo
, ir
->o
);
2081 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
2086 /* -- Trace setup --------------------------------------------------------- */
2088 /* Ensure there are enough stack slots for call arguments. */
2089 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
2091 IRRef args
[CCI_NARGS_MAX
];
2092 uint32_t i
, nargs
= (int)CCI_NARGS(ci
);
2093 int nslots
= 2, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
2094 asm_collectargs(as
, ir
, ci
, args
);
2095 for (i
= 0; i
< nargs
; i
++)
2096 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
2097 if (nfpr
> 0) nfpr
--; else nslots
= (nslots
+3) & ~1;
2099 if (ngpr
> 0) ngpr
--; else nslots
++;
2101 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
2102 as
->evenspill
= nslots
;
2103 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
2106 static void asm_setup_target(ASMState
*as
)
2108 asm_exitstub_setup(as
, as
->T
->nsnap
+ (as
->parent
? 1 : 0));
2111 /* -- Trace patching ------------------------------------------------------ */
2113 /* Patch exit jumps of existing machine code to a new target. */
2114 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
2116 MCode
*p
= T
->mcode
;
2117 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
2118 MCode
*px
= exitstub_trace_addr(T
, exitno
);
2119 MCode
*cstart
= NULL
;
2120 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
2122 for (; p
< pe
; p
++) {
2123 /* Look for exitstub branch, try to replace with branch to target. */
2125 if ((ins
& 0xfc000000u
) == 0x40000000u
&&
2126 ((ins
^ ((char *)px
-(char *)p
)) & 0xffffu
) == 0) {
2127 ptrdiff_t delta
= (char *)target
- (char *)p
;
2128 if (((ins
>> 16) & 3) == (CC_SO
&3)) {
2129 clearso
= sizeof(MCode
);
2130 delta
-= sizeof(MCode
);
2132 /* Many, but not all short-range branches can be patched directly. */
2133 if (((delta
+ 0x8000) >> 16) == 0) {
2134 *p
= (ins
& 0xffdf0000u
) | ((uint32_t)delta
& 0xffffu
) |
2135 ((delta
& 0x8000) * (PPCF_Y
/0x8000));
2136 if (!cstart
) cstart
= p
;
2138 } else if ((ins
& 0xfc000000u
) == PPCI_B
&&
2139 ((ins
^ ((char *)px
-(char *)p
)) & 0x03ffffffu
) == 0) {
2140 ptrdiff_t delta
= (char *)target
- (char *)p
;
2141 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2142 *p
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2143 if (!cstart
) cstart
= p
;
2146 { /* Always patch long-range branch in exit stub itself. */
2147 ptrdiff_t delta
= (char *)target
- (char *)px
- clearso
;
2148 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2149 *px
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2151 if (!cstart
) cstart
= px
;
2152 lj_mcode_sync(cstart
, px
+1);
2153 if (clearso
) { /* Extend the current trace. Ugly workaround. */
2154 MCode
*pp
= J
->cur
.mcode
;
2155 J
->cur
.szmcode
+= sizeof(MCode
);
2156 *--pp
= PPCI_MCRXR
; /* Clear SO flag. */
2158 lj_mcode_sync(pp
, pp
+1);
2160 lj_mcode_patch(J
, mcarea
, 1);