2 ** PPC IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate two source registers for three-operand instructions. */
22 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
24 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
25 Reg left
= irl
->r
, right
= irr
->r
;
26 if (ra_hasreg(left
)) {
29 right
= ra_allocref(as
, ir
->op2
, rset_exclude(allow
, left
));
32 } else if (ra_hasreg(right
)) {
34 left
= ra_allocref(as
, ir
->op1
, rset_exclude(allow
, right
));
35 } else if (ra_hashint(right
)) {
36 right
= ra_allocref(as
, ir
->op2
, allow
);
37 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(allow
, right
));
39 left
= ra_allocref(as
, ir
->op1
, allow
);
40 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(allow
, left
));
42 return left
| (right
<< 8);
45 /* -- Guard handling ------------------------------------------------------ */
47 /* Setup exit stubs after the end of each trace. */
48 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
51 MCode
*mxp
= as
->mctop
;
52 if (mxp
- (nexits
+ 3 + MCLIM_REDZONE
) < as
->mclim
)
54 /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
55 for (i
= nexits
-1; (int32_t)i
>= 0; i
--)
56 *--mxp
= PPCI_BL
|(((-3-i
)&0x00ffffffu
)<<2);
57 *--mxp
= PPCI_LI
|PPCF_T(RID_TMP
)|as
->T
->traceno
; /* Read by exit handler. */
59 *mxp
= PPCI_BL
|((((MCode
*)(void *)lj_vm_exit_handler
-mxp
)&0x00ffffffu
)<<2);
60 *--mxp
= PPCI_MFLR
|PPCF_T(RID_TMP
);
64 static MCode
*asm_exitstub_addr(ASMState
*as
, ExitNo exitno
)
66 /* Keep this in-sync with exitstub_trace_addr(). */
67 return as
->mctop
+ exitno
+ 3;
70 /* Emit conditional branch to exit for guard. */
71 static void asm_guardcc(ASMState
*as
, PPCCC cc
)
73 MCode
*target
= asm_exitstub_addr(as
, as
->snapno
);
75 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
77 *p
= PPCI_B
| (((target
-p
) & 0x00ffffffu
) << 2);
78 emit_condbranch(as
, PPCI_BC
, cc
^4, p
);
81 emit_condbranch(as
, PPCI_BC
, cc
, target
);
84 /* -- Operand fusion ------------------------------------------------------ */
86 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
87 #define CONFLICT_SEARCH_LIM 31
89 /* Check if there's no conflicting instruction between curins and ref. */
90 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
94 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
95 return 0; /* Give up, ref is too far away. */
97 if (ir
[i
].o
== conflict
)
98 return 0; /* Conflict found. */
99 return 1; /* Ok, no conflict. */
102 /* Fuse the array base of colocated arrays. */
103 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
106 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
107 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
108 return (int32_t)sizeof(GCtab
);
112 /* Indicates load/store indexed is ok. */
113 #define AHUREF_LSX ((int32_t)0x80000000)
115 /* Fuse array/hash/upvalue reference into register+offset operand. */
116 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
119 if (ra_noreg(ir
->r
)) {
120 if (ir
->o
== IR_AREF
) {
121 if (mayfuse(as
, ref
)) {
122 if (irref_isk(ir
->op2
)) {
123 IRRef tab
= IR(ir
->op1
)->op1
;
124 int32_t ofs
= asm_fuseabase(as
, tab
);
125 IRRef refa
= ofs
? tab
: ir
->op1
;
126 ofs
+= 8*IR(ir
->op2
)->i
;
129 return ra_alloc1(as
, refa
, allow
);
132 if (*ofsp
== AHUREF_LSX
) {
133 Reg base
= ra_alloc1(as
, ir
->op1
, allow
);
134 Reg idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
135 return base
| (idx
<< 8);
138 } else if (ir
->o
== IR_HREFK
) {
139 if (mayfuse(as
, ref
)) {
140 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
143 return ra_alloc1(as
, ir
->op1
, allow
);
146 } else if (ir
->o
== IR_UREFC
) {
147 if (irref_isk(ir
->op1
)) {
148 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
149 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
150 int32_t jgl
= (intptr_t)J2G(as
->J
);
151 if ((uint32_t)(ofs
-jgl
) < 65536) {
152 *ofsp
= ofs
-jgl
-32768;
155 *ofsp
= (int16_t)ofs
;
156 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
162 return ra_alloc1(as
, ref
, allow
);
165 /* Fuse XLOAD/XSTORE reference into load/store operand. */
166 static void asm_fusexref(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
167 RegSet allow
, int32_t ofs
)
171 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
172 if (ir
->o
== IR_ADD
) {
174 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
177 } else if (ofs
== 0) {
178 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
179 right
= (left
>> 8); left
&= 255;
180 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
183 } else if (ir
->o
== IR_STRREF
) {
184 lua_assert(ofs
== 0);
185 ofs
= (int32_t)sizeof(GCstr
);
186 if (irref_isk(ir
->op2
)) {
187 ofs
+= IR(ir
->op2
)->i
;
189 } else if (irref_isk(ir
->op1
)) {
190 ofs
+= IR(ir
->op1
)->i
;
193 /* NYI: Fuse ADD with constant. */
194 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, allow
);
195 right
= (left
>> 8); left
&= 255;
196 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(allow
, left
), right
));
197 emit_fai(as
, pi
, rt
, tmp
, ofs
);
198 emit_tab(as
, PPCI_ADD
, tmp
, left
, right
);
201 if (!checki16(ofs
)) {
202 Reg left
= ra_alloc1(as
, ref
, allow
);
203 Reg right
= ra_allock(as
, ofs
, rset_exclude(allow
, left
));
204 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
209 base
= ra_alloc1(as
, ref
, allow
);
210 emit_fai(as
, pi
, rt
, base
, ofs
);
213 /* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
214 static void asm_fusexrefx(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
217 IRIns
*ira
= IR(ref
);
219 if (canfuse(as
, ira
) && ira
->o
== IR_ADD
&& ra_noreg(ira
->r
)) {
220 left
= ra_alloc2(as
, ira
, allow
);
221 right
= (left
>> 8); left
&= 255;
223 right
= ra_alloc1(as
, ref
, allow
);
226 emit_tab(as
, pi
, rt
, left
, right
);
229 /* Fuse to multiply-add/sub instruction. */
230 static int asm_fusemadd(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pir
)
232 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
235 ((mayfuse(as
, lref
) && (irm
= IR(lref
), irm
->o
== IR_MUL
) &&
237 (mayfuse(as
, rref
) && (irm
= IR(rref
), irm
->o
== IR_MUL
) &&
238 (rref
= lref
, pi
= pir
, ra_noreg(irm
->r
))))) {
239 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
240 Reg add
= ra_alloc1(as
, rref
, RSET_FPR
);
241 Reg right
, left
= ra_alloc2(as
, irm
, rset_exclude(RSET_FPR
, add
));
242 right
= (left
>> 8); left
&= 255;
243 emit_facb(as
, pi
, dest
, left
, right
, add
);
249 /* -- Calls --------------------------------------------------------------- */
251 /* Generate a call to a C function. */
252 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
254 uint32_t n
, nargs
= CCI_XNARGS(ci
);
256 Reg gpr
= REGARG_FIRSTGPR
, fpr
= REGARG_FIRSTFPR
;
257 if ((void *)ci
->func
)
258 emit_call(as
, (void *)ci
->func
);
259 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
263 if (irt_isfp(ir
->t
)) {
264 if (fpr
<= REGARG_LASTFPR
) {
265 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
266 ra_leftov(as
, fpr
, ref
);
269 Reg r
= ra_alloc1(as
, ref
, RSET_FPR
);
270 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
271 emit_spstore(as
, ir
, r
, ofs
);
272 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
275 if (gpr
<= REGARG_LASTGPR
) {
276 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
277 ra_leftov(as
, gpr
, ref
);
280 Reg r
= ra_alloc1(as
, ref
, RSET_GPR
);
281 emit_spstore(as
, ir
, r
, ofs
);
286 if (gpr
<= REGARG_LASTGPR
)
293 if ((ci
->flags
& CCI_VARARG
)) /* Vararg calls need to know about FPR use. */
294 emit_tab(as
, fpr
== REGARG_FIRSTFPR
? PPCI_CRXOR
: PPCI_CREQV
, 6, 6, 6);
297 /* Setup result reg/sp for call. Evict scratch regs. */
298 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
300 RegSet drop
= RSET_SCRATCH
;
301 int hiop
= ((ir
+1)->o
== IR_HIOP
);
302 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
304 if (ra_hasreg(ir
->r
))
305 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
306 if (hiop
&& ra_hasreg((ir
+1)->r
))
307 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
308 ra_evictset(as
, drop
); /* Evictions must be performed first. */
310 lua_assert(!irt_ispri(ir
->t
));
311 if (irt_isfp(ir
->t
)) {
312 if ((ci
->flags
& CCI_CASTU64
)) {
313 /* Use spill slot or temp slots. */
314 int32_t ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
316 if (ra_hasreg(dest
)) {
318 ra_modified(as
, dest
);
319 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, ofs
);
321 emit_tai(as
, PPCI_STW
, RID_RETHI
, RID_SP
, ofs
);
322 emit_tai(as
, PPCI_STW
, RID_RETLO
, RID_SP
, ofs
+4);
324 ra_destreg(as
, ir
, RID_FPRET
);
329 ra_destreg(as
, ir
, RID_RET
);
334 static void asm_callx(ASMState
*as
, IRIns
*ir
)
336 IRRef args
[CCI_NARGS_MAX
*2];
340 ci
.flags
= asm_callx_flags(as
, ir
);
341 asm_collectargs(as
, ir
, &ci
, args
);
342 asm_setupresult(as
, ir
, &ci
);
343 func
= ir
->op2
; irf
= IR(func
);
344 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
345 if (irref_isk(func
)) { /* Call to constant address. */
346 ci
.func
= (ASMFunction
)(void *)(irf
->i
);
347 } else { /* Need a non-argument register for indirect calls. */
348 RegSet allow
= RSET_GPR
& ~RSET_RANGE(RID_R0
, REGARG_LASTGPR
+1);
349 Reg freg
= ra_alloc1(as
, func
, allow
);
350 *--as
->mcp
= PPCI_BCTRL
;
351 *--as
->mcp
= PPCI_MTCTR
| PPCF_T(freg
);
352 ci
.func
= (ASMFunction
)(void *)0;
354 asm_gencall(as
, &ci
, args
);
357 /* -- Returns ------------------------------------------------------------- */
359 /* Return to lower frame. Guard that it goes to the right spot. */
360 static void asm_retf(ASMState
*as
, IRIns
*ir
)
362 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
363 void *pc
= ir_kptr(IR(ir
->op2
));
364 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
365 as
->topslot
-= (BCReg
)delta
;
366 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
367 irt_setmark(IR(REF_BASE
)->t
); /* Children must not coalesce with BASE reg. */
368 emit_setgl(as
, base
, jit_base
);
369 emit_addptr(as
, base
, -8*delta
);
370 asm_guardcc(as
, CC_NE
);
371 emit_ab(as
, PPCI_CMPW
, RID_TMP
,
372 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
373 emit_tai(as
, PPCI_LWZ
, RID_TMP
, base
, -8);
376 /* -- Type conversions ---------------------------------------------------- */
378 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
380 RegSet allow
= RSET_FPR
;
381 Reg tmp
= ra_scratch(as
, rset_clear(allow
, left
));
382 Reg fbias
= ra_scratch(as
, rset_clear(allow
, tmp
));
383 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
384 Reg hibias
= ra_allock(as
, 0x43300000, rset_exclude(RSET_GPR
, dest
));
385 asm_guardcc(as
, CC_NE
);
386 emit_fab(as
, PPCI_FCMPU
, 0, tmp
, left
);
387 emit_fab(as
, PPCI_FSUB
, tmp
, tmp
, fbias
);
388 emit_fai(as
, PPCI_LFD
, tmp
, RID_SP
, SPOFS_TMP
);
389 emit_tai(as
, PPCI_STW
, RID_TMP
, RID_SP
, SPOFS_TMPLO
);
390 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
391 emit_asi(as
, PPCI_XORIS
, RID_TMP
, dest
, 0x8000);
392 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
393 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
394 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
396 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
397 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
400 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
402 RegSet allow
= RSET_FPR
;
403 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
404 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
405 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
406 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
407 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
408 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
409 emit_fab(as
, PPCI_FADD
, tmp
, left
, right
);
412 static void asm_conv(ASMState
*as
, IRIns
*ir
)
414 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
415 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
416 IRRef lref
= ir
->op1
;
417 lua_assert(irt_type(ir
->t
) != st
);
418 lua_assert(!(irt_isint64(ir
->t
) ||
419 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
420 if (irt_isfp(ir
->t
)) {
421 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
422 if (stfp
) { /* FP to FP conversion. */
423 if (st
== IRT_NUM
) /* double -> float conversion. */
424 emit_fb(as
, PPCI_FRSP
, dest
, ra_alloc1(as
, lref
, RSET_FPR
));
425 else /* float -> double conversion is a no-op on PPC. */
426 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
427 } else { /* Integer to FP conversion. */
428 /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
429 /* IRT_U32: Bias with 2^52, subtract 2^52. */
430 RegSet allow
= RSET_GPR
;
431 Reg left
= ra_alloc1(as
, lref
, allow
);
432 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, left
));
433 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
435 if (irt_isfloat(ir
->t
)) emit_fb(as
, PPCI_FRSP
, dest
, dest
);
436 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
437 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
438 kbias
= (const float *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000));
439 if (st
== IRT_U32
) kbias
++;
440 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31), (void *)kbias
,
441 rset_clear(allow
, hibias
));
442 emit_tai(as
, PPCI_STW
, st
== IRT_U32
? left
: RID_TMP
,
443 RID_SP
, SPOFS_TMPLO
);
444 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
445 if (st
!= IRT_U32
) emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, 0x8000);
447 } else if (stfp
) { /* FP to integer conversion. */
448 if (irt_isguard(ir
->t
)) {
449 /* Checked conversions are only supported from number to int. */
450 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
451 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
453 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
454 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
455 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
456 if (irt_isu32(ir
->t
)) {
457 /* Convert both x and x-2^31 to int and merge results. */
458 Reg tmpi
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
459 emit_asb(as
, PPCI_OR
, dest
, dest
, tmpi
); /* Select with mask idiom. */
460 emit_asb(as
, PPCI_AND
, tmpi
, tmpi
, RID_TMP
);
461 emit_asb(as
, PPCI_ANDC
, dest
, dest
, RID_TMP
);
462 emit_tai(as
, PPCI_LWZ
, tmpi
, RID_SP
, SPOFS_TMPLO
); /* tmp = (int)(x) */
463 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, 0x8000); /* dest += 2^31 */
464 emit_asb(as
, PPCI_SRAWI
, RID_TMP
, dest
, 31); /* mask = -(dest < 0) */
465 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
466 emit_tai(as
, PPCI_LWZ
, dest
,
467 RID_SP
, SPOFS_TMPLO
); /* dest = (int)(x-2^31) */
468 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
469 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
470 emit_fb(as
, PPCI_FCTIWZ
, tmp
, tmp
);
471 emit_fab(as
, PPCI_FSUB
, tmp
, left
, tmp
);
472 emit_lsptr(as
, PPCI_LFS
, (tmp
& 31),
473 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,00000000)),
476 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
477 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
478 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
482 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
483 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
484 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
485 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
486 if ((ir
->op2
& IRCONV_SEXT
))
487 emit_as(as
, st
== IRT_I8
? PPCI_EXTSB
: PPCI_EXTSH
, dest
, left
);
489 emit_rot(as
, PPCI_RLWINM
, dest
, left
, 0, st
== IRT_U8
? 24 : 16, 31);
490 } else { /* 32/64 bit integer conversions. */
491 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
492 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
497 static void asm_strto(ASMState
*as
, IRIns
*ir
)
499 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
502 RegSet drop
= RSET_SCRATCH
;
503 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
504 ra_evictset(as
, drop
);
505 asm_guardcc(as
, CC_EQ
);
506 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0); /* Test return status. */
507 args
[0] = ir
->op1
; /* GCstr *str */
508 args
[1] = ASMREF_TMP1
; /* TValue *n */
509 asm_gencall(as
, ci
, args
);
510 /* Store the result to the spill slot or temp slots. */
511 ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
512 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_SP
, ofs
);
515 /* -- Memory references --------------------------------------------------- */
517 /* Get pointer to TValue. */
518 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
521 if (irt_isnum(ir
->t
)) {
522 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
523 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
524 else /* Otherwise force a spill and use the spill slot. */
525 emit_tai(as
, PPCI_ADDI
, dest
, RID_SP
, ra_spill(as
, ir
));
527 /* Otherwise use g->tmptv to hold the TValue. */
528 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
530 emit_tai(as
, PPCI_ADDI
, dest
, RID_JGL
, offsetof(global_State
, tmptv
)-32768);
531 if (!irt_ispri(ir
->t
)) {
532 Reg src
= ra_alloc1(as
, ref
, allow
);
533 emit_setgl(as
, src
, tmptv
.gcr
);
535 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
536 emit_setgl(as
, type
, tmptv
.it
);
540 static void asm_aref(ASMState
*as
, IRIns
*ir
)
542 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
544 if (irref_isk(ir
->op2
)) {
545 IRRef tab
= IR(ir
->op1
)->op1
;
546 int32_t ofs
= asm_fuseabase(as
, tab
);
547 IRRef refa
= ofs
? tab
: ir
->op1
;
548 ofs
+= 8*IR(ir
->op2
)->i
;
550 base
= ra_alloc1(as
, refa
, RSET_GPR
);
551 emit_tai(as
, PPCI_ADDI
, dest
, base
, ofs
);
555 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
556 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
557 emit_tab(as
, PPCI_ADD
, dest
, RID_TMP
, base
);
558 emit_slwi(as
, RID_TMP
, idx
, 3);
561 /* Inlined hash lookup. Specialized for key type and for const keys.
562 ** The equivalent C code is:
563 ** Node *n = hashkey(t, key);
565 ** if (lj_obj_equal(&n->key, key)) return &n->val;
566 ** } while ((n = nextnode(n)));
569 static void asm_href(ASMState
*as
, IRIns
*ir
, IROp merge
)
571 RegSet allow
= RSET_GPR
;
572 int destused
= ra_used(ir
);
573 Reg dest
= ra_dest(as
, ir
, allow
);
574 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
575 Reg key
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
576 Reg tisnum
= RID_NONE
, tmpnum
= RID_NONE
;
577 IRRef refkey
= ir
->op2
;
578 IRIns
*irkey
= IR(refkey
);
579 IRType1 kt
= irkey
->t
;
581 MCLabel l_end
, l_loop
, l_next
;
583 rset_clear(allow
, tab
);
585 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
586 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
587 tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
588 rset_clear(allow
, tisnum
);
589 } else if (!irt_ispri(kt
)) {
590 key
= ra_alloc1(as
, refkey
, allow
);
591 rset_clear(allow
, key
);
593 tmp2
= ra_scratch(as
, allow
);
594 rset_clear(allow
, tmp2
);
596 /* Key not found in chain: jump to exit (if merged) or load niltv. */
597 l_end
= emit_label(as
);
600 asm_guardcc(as
, CC_EQ
);
602 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
604 /* Follow hash chain until the end. */
606 emit_ai(as
, PPCI_CMPWI
, dest
, 0);
607 emit_tai(as
, PPCI_LWZ
, dest
, dest
, (int32_t)offsetof(Node
, next
));
608 l_next
= emit_label(as
);
610 /* Type and value comparison. */
612 asm_guardcc(as
, CC_EQ
);
614 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
616 emit_fab(as
, PPCI_FCMPU
, 0, tmpnum
, key
);
617 emit_condbranch(as
, PPCI_BC
, CC_GE
, l_next
);
618 emit_ab(as
, PPCI_CMPLW
, tmp1
, tisnum
);
619 emit_fai(as
, PPCI_LFD
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
621 if (!irt_ispri(kt
)) {
622 emit_ab(as
, PPCI_CMPW
, tmp2
, key
);
623 emit_condbranch(as
, PPCI_BC
, CC_NE
, l_next
);
625 emit_ai(as
, PPCI_CMPWI
, tmp1
, irt_toitype(irkey
->t
));
627 emit_tai(as
, PPCI_LWZ
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
629 emit_tai(as
, PPCI_LWZ
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
630 *l_loop
= PPCI_BC
| PPCF_Y
| PPCF_CC(CC_NE
) |
631 (((char *)as
->mcp
-(char *)l_loop
) & 0xffffu
);
633 /* Load main position relative to tab->node into dest. */
634 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
636 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
639 if (irref_isk(refkey
))
640 tmphash
= ra_allock(as
, khash
, allow
);
641 emit_tab(as
, PPCI_ADD
, dest
, dest
, tmp1
);
642 emit_tai(as
, PPCI_MULLI
, tmp1
, tmp1
, sizeof(Node
));
643 emit_asb(as
, PPCI_AND
, tmp1
, tmp2
, tmphash
);
644 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
645 emit_tai(as
, PPCI_LWZ
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
646 if (irref_isk(refkey
)) {
648 } else if (irt_isstr(kt
)) {
649 emit_tai(as
, PPCI_LWZ
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
650 } else { /* Must match with hash*() in lj_tab.c. */
651 emit_tab(as
, PPCI_SUBF
, tmp1
, tmp2
, tmp1
);
652 emit_rotlwi(as
, tmp2
, tmp2
, HASH_ROT3
);
653 emit_asb(as
, PPCI_XOR
, tmp1
, tmp1
, tmp2
);
654 emit_rotlwi(as
, tmp1
, tmp1
, (HASH_ROT2
+HASH_ROT1
)&31);
655 emit_tab(as
, PPCI_SUBF
, tmp2
, dest
, tmp2
);
657 int32_t ofs
= ra_spill(as
, irkey
);
658 emit_asb(as
, PPCI_XOR
, tmp2
, tmp2
, tmp1
);
659 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
660 emit_tab(as
, PPCI_ADD
, tmp1
, tmp1
, tmp1
);
661 emit_tai(as
, PPCI_LWZ
, tmp2
, RID_SP
, ofs
+4);
662 emit_tai(as
, PPCI_LWZ
, tmp1
, RID_SP
, ofs
);
664 emit_asb(as
, PPCI_XOR
, tmp2
, key
, tmp1
);
665 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
666 emit_tai(as
, PPCI_ADDI
, tmp1
, tmp2
, HASH_BIAS
);
667 emit_tai(as
, PPCI_ADDIS
, tmp2
, key
, (HASH_BIAS
+ 32768)>>16);
673 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
675 IRIns
*kslot
= IR(ir
->op2
);
676 IRIns
*irkey
= IR(kslot
->op1
);
677 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
678 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
679 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
680 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
681 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
682 RegSet allow
= rset_exclude(RSET_GPR
, node
);
683 lua_assert(ofs
% sizeof(Node
) == 0);
686 rset_clear(allow
, dest
);
687 kofs
= (int32_t)offsetof(Node
, key
);
688 } else if (ra_hasreg(dest
)) {
689 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
691 asm_guardcc(as
, CC_NE
);
692 if (!irt_ispri(irkey
->t
)) {
693 key
= ra_scratch(as
, allow
);
694 rset_clear(allow
, key
);
696 rset_clear(allow
, type
);
697 if (irt_isnum(irkey
->t
)) {
698 emit_cmpi(as
, key
, (int32_t)ir_knum(irkey
)->u32
.lo
);
699 asm_guardcc(as
, CC_NE
);
700 emit_cmpi(as
, type
, (int32_t)ir_knum(irkey
)->u32
.hi
);
702 if (ra_hasreg(key
)) {
703 emit_cmpi(as
, key
, irkey
->i
); /* May use RID_TMP, i.e. type. */
704 asm_guardcc(as
, CC_NE
);
706 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(irkey
->t
));
708 if (ra_hasreg(key
)) emit_tai(as
, PPCI_LWZ
, key
, idx
, kofs
+4);
709 emit_tai(as
, PPCI_LWZ
, type
, idx
, kofs
);
711 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (ofs
+ 32768) >> 16);
712 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
716 static void asm_uref(ASMState
*as
, IRIns
*ir
)
718 /* NYI: Check that UREFO is still open and not aliasing a slot. */
719 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
720 if (irref_isk(ir
->op1
)) {
721 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
722 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
723 emit_lsptr(as
, PPCI_LWZ
, dest
, v
, RSET_GPR
);
725 Reg uv
= ra_scratch(as
, RSET_GPR
);
726 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
727 if (ir
->o
== IR_UREFC
) {
728 asm_guardcc(as
, CC_NE
);
729 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, 1);
730 emit_tai(as
, PPCI_ADDI
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
731 emit_tai(as
, PPCI_LBZ
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
733 emit_tai(as
, PPCI_LWZ
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
735 emit_tai(as
, PPCI_LWZ
, uv
, func
,
736 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
740 static void asm_fref(ASMState
*as
, IRIns
*ir
)
742 UNUSED(as
); UNUSED(ir
);
743 lua_assert(!ra_used(ir
));
746 static void asm_strref(ASMState
*as
, IRIns
*ir
)
748 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
749 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
750 int32_t ofs
= (int32_t)sizeof(GCstr
);
752 if (irref_isk(ref
)) {
753 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
754 } else if (!irref_isk(refk
)) {
755 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
756 IRIns
*irr
= IR(ir
->op2
);
757 if (ra_hasreg(irr
->r
)) {
758 ra_noweak(as
, irr
->r
);
760 } else if (mayfuse(as
, irr
->op2
) &&
761 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
762 checki16(ofs
+ IR(irr
->op2
)->i
)) {
763 ofs
+= IR(irr
->op2
)->i
;
764 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
766 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
768 emit_tai(as
, PPCI_ADDI
, dest
, dest
, ofs
);
769 emit_tab(as
, PPCI_ADD
, dest
, left
, right
);
772 r
= ra_alloc1(as
, ref
, RSET_GPR
);
775 emit_tai(as
, PPCI_ADDI
, dest
, r
, ofs
);
777 emit_tab(as
, PPCI_ADD
, dest
, r
,
778 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
781 /* -- Loads and stores ---------------------------------------------------- */
783 static PPCIns
asm_fxloadins(IRIns
*ir
)
785 switch (irt_type(ir
->t
)) {
786 case IRT_I8
: return PPCI_LBZ
; /* Needs sign-extension. */
787 case IRT_U8
: return PPCI_LBZ
;
788 case IRT_I16
: return PPCI_LHA
;
789 case IRT_U16
: return PPCI_LHZ
;
790 case IRT_NUM
: return PPCI_LFD
;
791 case IRT_FLOAT
: return PPCI_LFS
;
792 default: return PPCI_LWZ
;
796 static PPCIns
asm_fxstoreins(IRIns
*ir
)
798 switch (irt_type(ir
->t
)) {
799 case IRT_I8
: case IRT_U8
: return PPCI_STB
;
800 case IRT_I16
: case IRT_U16
: return PPCI_STH
;
801 case IRT_NUM
: return PPCI_STFD
;
802 case IRT_FLOAT
: return PPCI_STFS
;
803 default: return PPCI_STW
;
807 static void asm_fload(ASMState
*as
, IRIns
*ir
)
809 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
810 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
811 PPCIns pi
= asm_fxloadins(ir
);
813 if (ir
->op2
== IRFL_TAB_ARRAY
) {
814 ofs
= asm_fuseabase(as
, ir
->op1
);
815 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
816 emit_tai(as
, PPCI_ADDI
, dest
, idx
, ofs
);
820 ofs
= field_ofs
[ir
->op2
];
821 lua_assert(!irt_isi8(ir
->t
));
822 emit_tai(as
, pi
, dest
, idx
, ofs
);
825 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
827 if (ir
->r
!= RID_SINK
) {
828 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
829 IRIns
*irf
= IR(ir
->op1
);
830 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
831 int32_t ofs
= field_ofs
[irf
->op2
];
832 PPCIns pi
= asm_fxstoreins(ir
);
833 emit_tai(as
, pi
, src
, idx
, ofs
);
837 static void asm_xload(ASMState
*as
, IRIns
*ir
)
839 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
840 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
842 emit_as(as
, PPCI_EXTSB
, dest
, dest
);
843 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
846 static void asm_xstore_(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
849 if (ir
->r
== RID_SINK
)
851 if (ofs
== 0 && mayfuse(as
, ir
->op2
) && (irb
= IR(ir
->op2
))->o
== IR_BSWAP
&&
852 ra_noreg(irb
->r
) && (irt_isint(ir
->t
) || irt_isu32(ir
->t
))) {
853 /* Fuse BSWAP with XSTORE to stwbrx. */
854 Reg src
= ra_alloc1(as
, irb
->op1
, RSET_GPR
);
855 asm_fusexrefx(as
, PPCI_STWBRX
, src
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
857 Reg src
= ra_alloc1(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
858 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
859 rset_exclude(RSET_GPR
, src
), ofs
);
863 #define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
865 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
868 Reg dest
= RID_NONE
, type
= RID_TMP
, tmp
= RID_TMP
, idx
;
869 RegSet allow
= RSET_GPR
;
870 int32_t ofs
= AHUREF_LSX
;
872 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
873 if (!irt_isnum(t
)) ofs
= 0;
874 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
875 rset_clear(allow
, dest
);
877 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
879 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, rset_exclude(allow
, idx
));
880 asm_guardcc(as
, CC_GE
);
881 emit_ab(as
, PPCI_CMPLW
, type
, tisnum
);
882 if (ra_hasreg(dest
)) {
883 if (ofs
== AHUREF_LSX
) {
884 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
,
885 (idx
&255)), (idx
>>8)));
886 emit_fab(as
, PPCI_LFDX
, dest
, (idx
&255), tmp
);
888 emit_fai(as
, PPCI_LFD
, dest
, idx
, ofs
);
892 asm_guardcc(as
, CC_NE
);
893 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(t
));
894 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, idx
, ofs
+4);
896 if (ofs
== AHUREF_LSX
) {
897 emit_tab(as
, PPCI_LWZX
, type
, (idx
&255), tmp
);
898 emit_slwi(as
, tmp
, (idx
>>8), 3);
900 emit_tai(as
, PPCI_LWZ
, type
, idx
, ofs
);
904 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
906 RegSet allow
= RSET_GPR
;
907 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
908 int32_t ofs
= AHUREF_LSX
;
909 if (ir
->r
== RID_SINK
)
911 if (irt_isnum(ir
->t
)) {
912 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
914 if (!irt_ispri(ir
->t
)) {
915 src
= ra_alloc1(as
, ir
->op2
, allow
);
916 rset_clear(allow
, src
);
919 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
920 rset_clear(allow
, type
);
922 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
923 if (irt_isnum(ir
->t
)) {
924 if (ofs
== AHUREF_LSX
) {
925 emit_fab(as
, PPCI_STFDX
, src
, (idx
&255), RID_TMP
);
926 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
928 emit_fai(as
, PPCI_STFD
, src
, idx
, ofs
);
932 emit_tai(as
, PPCI_STW
, src
, idx
, ofs
+4);
933 if (ofs
== AHUREF_LSX
) {
934 emit_tab(as
, PPCI_STWX
, type
, (idx
&255), RID_TMP
);
935 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
937 emit_tai(as
, PPCI_STW
, type
, idx
, ofs
);
942 static void asm_sload(ASMState
*as
, IRIns
*ir
)
944 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 0 : 4);
946 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
947 RegSet allow
= RSET_GPR
;
948 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
949 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
950 lua_assert(LJ_DUALNUM
||
951 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
952 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
953 dest
= ra_scratch(as
, RSET_FPR
);
954 asm_tointg(as
, ir
, dest
);
955 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
956 } else if (ra_used(ir
)) {
957 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
958 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
959 rset_clear(allow
, dest
);
960 base
= ra_alloc1(as
, REF_BASE
, allow
);
961 rset_clear(allow
, base
);
962 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
964 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
965 dest
= ra_scratch(as
, RSET_FPR
);
966 emit_fai(as
, PPCI_STFD
, dest
, RID_SP
, SPOFS_TMP
);
967 emit_fb(as
, PPCI_FCTIWZ
, dest
, dest
);
968 t
.irt
= IRT_NUM
; /* Check for original type. */
970 Reg tmp
= ra_scratch(as
, allow
);
971 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, tmp
));
972 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
973 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
974 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
975 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
976 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
977 rset_clear(allow
, hibias
));
978 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPLO
);
979 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
980 emit_asi(as
, PPCI_XORIS
, tmp
, tmp
, 0x8000);
982 t
.irt
= IRT_INT
; /* Check for original type. */
987 base
= ra_alloc1(as
, REF_BASE
, allow
);
988 rset_clear(allow
, base
);
991 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
992 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
993 asm_guardcc(as
, CC_GE
);
994 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tisnum
);
997 if (ra_hasreg(dest
)) emit_fai(as
, PPCI_LFD
, dest
, base
, ofs
-4);
999 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1000 asm_guardcc(as
, CC_NE
);
1001 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, irt_toitype(t
));
1004 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, base
, ofs
);
1006 if (ra_hasreg(type
)) emit_tai(as
, PPCI_LWZ
, type
, base
, ofs
-4);
1009 /* -- Allocations --------------------------------------------------------- */
1012 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1014 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1015 CTypeID id
= (CTypeID
)IR(ir
->op1
)->i
;
1017 CTInfo info
= lj_ctype_info(cts
, id
, &sz
);
1018 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1020 RegSet drop
= RSET_SCRATCH
;
1021 lua_assert(sz
!= CTSIZE_INVALID
|| (ir
->o
== IR_CNEW
&& ir
->op2
!= REF_NIL
));
1024 if (ra_hasreg(ir
->r
))
1025 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1026 ra_evictset(as
, drop
);
1028 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1030 /* Initialize immutable cdata object. */
1031 if (ir
->o
== IR_CNEWI
) {
1032 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1033 int32_t ofs
= sizeof(GCcdata
);
1034 lua_assert(sz
== 4 || sz
== 8);
1037 lua_assert((ir
+1)->o
== IR_HIOP
);
1040 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1041 emit_tai(as
, PPCI_STW
, r
, RID_RET
, ofs
);
1042 rset_clear(allow
, r
);
1043 if (ofs
== sizeof(GCcdata
)) break;
1046 } else if (ir
->op2
!= REF_NIL
) { /* Create VLA/VLS/aligned cdata. */
1047 ci
= &lj_ir_callinfo
[IRCALL_lj_cdata_newv
];
1048 args
[0] = ASMREF_L
; /* lua_State *L */
1049 args
[1] = ir
->op1
; /* CTypeID id */
1050 args
[2] = ir
->op2
; /* CTSize sz */
1051 args
[3] = ASMREF_TMP1
; /* CTSize align */
1052 asm_gencall(as
, ci
, args
);
1053 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP1
), (int32_t)ctype_align(info
));
1057 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1058 emit_tai(as
, PPCI_STB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1059 emit_tai(as
, PPCI_STH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1060 emit_ti(as
, PPCI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1061 emit_ti(as
, PPCI_LI
, RID_TMP
, id
); /* Lower 16 bit used. Sign-ext ok. */
1062 args
[0] = ASMREF_L
; /* lua_State *L */
1063 args
[1] = ASMREF_TMP1
; /* MSize size */
1064 asm_gencall(as
, ci
, args
);
1065 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1066 ra_releasetmp(as
, ASMREF_TMP1
));
1069 #define asm_cnew(as, ir) ((void)0)
1072 /* -- Write barriers ------------------------------------------------------ */
1074 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1076 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1077 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1079 MCLabel l_end
= emit_label(as
);
1080 emit_tai(as
, PPCI_STW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1081 emit_tai(as
, PPCI_STB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1082 emit_setgl(as
, tab
, gc
.grayagain
);
1083 lua_assert(LJ_GC_BLACK
== 0x04);
1084 emit_rot(as
, PPCI_RLWINM
, mark
, mark
, 0, 30, 28); /* Clear black bit. */
1085 emit_getgl(as
, link
, gc
.grayagain
);
1086 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1087 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, mark
, LJ_GC_BLACK
);
1088 emit_tai(as
, PPCI_LBZ
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1091 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1093 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1097 /* No need for other object barriers (yet). */
1098 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1099 ra_evictset(as
, RSET_SCRATCH
);
1100 l_end
= emit_label(as
);
1101 args
[0] = ASMREF_TMP1
; /* global_State *g */
1102 args
[1] = ir
->op1
; /* TValue *tv */
1103 asm_gencall(as
, ci
, args
);
1104 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1105 obj
= IR(ir
->op1
)->r
;
1106 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1107 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1108 emit_asi(as
, PPCI_ANDIDOT
, tmp
, tmp
, LJ_GC_BLACK
);
1109 emit_condbranch(as
, PPCI_BC
, CC_EQ
, l_end
);
1110 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1111 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1112 emit_tai(as
, PPCI_LBZ
, tmp
, obj
,
1113 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1114 emit_tai(as
, PPCI_LBZ
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1117 /* -- Arithmetic and logic operations ------------------------------------- */
1119 static void asm_fparith(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1121 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1122 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1123 right
= (left
>> 8); left
&= 255;
1124 if (pi
== PPCI_FMUL
)
1125 emit_fac(as
, pi
, dest
, left
, right
);
1127 emit_fab(as
, pi
, dest
, left
, right
);
1130 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1132 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1133 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1134 emit_fb(as
, pi
, dest
, left
);
1137 static void asm_fpmath(ASMState
*as
, IRIns
*ir
)
1139 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
1141 if (ir
->op2
== IRFPM_SQRT
&& (as
->flags
& JIT_F_SQRT
))
1142 asm_fpunary(as
, ir
, PPCI_FSQRT
);
1144 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1147 static void asm_add(ASMState
*as
, IRIns
*ir
)
1149 if (irt_isnum(ir
->t
)) {
1150 if (!asm_fusemadd(as
, ir
, PPCI_FMADD
, PPCI_FMADD
))
1151 asm_fparith(as
, ir
, PPCI_FADD
);
1153 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1154 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1156 if (irref_isk(ir
->op2
)) {
1157 int32_t k
= IR(ir
->op2
)->i
;
1160 /* May fail due to spills/restores above, but simplifies the logic. */
1161 if (as
->flagmcp
== as
->mcp
) {
1166 emit_tai(as
, pi
, dest
, left
, k
);
1168 } else if ((k
& 0xffff) == 0) {
1169 emit_tai(as
, PPCI_ADDIS
, dest
, left
, (k
>> 16));
1171 } else if (!as
->sectref
) {
1172 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (k
+ 32768) >> 16);
1173 emit_tai(as
, PPCI_ADDI
, dest
, left
, k
);
1178 /* May fail due to spills/restores above, but simplifies the logic. */
1179 if (as
->flagmcp
== as
->mcp
) {
1184 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1185 emit_tab(as
, pi
, dest
, left
, right
);
1189 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1191 if (irt_isnum(ir
->t
)) {
1192 if (!asm_fusemadd(as
, ir
, PPCI_FMSUB
, PPCI_FNMSUB
))
1193 asm_fparith(as
, ir
, PPCI_FSUB
);
1195 PPCIns pi
= PPCI_SUBF
;
1196 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1198 if (irref_isk(ir
->op1
)) {
1199 int32_t k
= IR(ir
->op1
)->i
;
1201 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1202 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1206 /* May fail due to spills/restores above, but simplifies the logic. */
1207 if (as
->flagmcp
== as
->mcp
) {
1212 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1213 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1214 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1218 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1220 if (irt_isnum(ir
->t
)) {
1221 asm_fparith(as
, ir
, PPCI_FMUL
);
1223 PPCIns pi
= PPCI_MULLW
;
1224 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1225 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1226 if (irref_isk(ir
->op2
)) {
1227 int32_t k
= IR(ir
->op2
)->i
;
1229 emit_tai(as
, PPCI_MULLI
, dest
, left
, k
);
1233 /* May fail due to spills/restores above, but simplifies the logic. */
1234 if (as
->flagmcp
== as
->mcp
) {
1239 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1240 emit_tab(as
, pi
, dest
, left
, right
);
1244 #define asm_div(as, ir) asm_fparith(as, ir, PPCI_FDIV)
1245 #define asm_mod(as, ir) asm_callid(as, ir, IRCALL_lj_vm_modi)
1246 #define asm_pow(as, ir) asm_callid(as, ir, IRCALL_lj_vm_powi)
1248 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1250 if (irt_isnum(ir
->t
)) {
1251 asm_fpunary(as
, ir
, PPCI_FNEG
);
1254 PPCIns pi
= PPCI_NEG
;
1255 if (as
->flagmcp
== as
->mcp
) {
1260 dest
= ra_dest(as
, ir
, RSET_GPR
);
1261 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1262 emit_tab(as
, pi
, dest
, left
, 0);
1266 #define asm_abs(as, ir) asm_fpunary(as, ir, PPCI_FABS)
1267 #define asm_atan2(as, ir) asm_callid(as, ir, IRCALL_atan2)
1268 #define asm_ldexp(as, ir) asm_callid(as, ir, IRCALL_ldexp)
1270 static void asm_arithov(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1272 Reg dest
, left
, right
;
1273 if (as
->flagmcp
== as
->mcp
) {
1277 asm_guardcc(as
, CC_SO
);
1278 dest
= ra_dest(as
, ir
, RSET_GPR
);
1279 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1280 right
= (left
>> 8); left
&= 255;
1281 if (pi
== PPCI_SUBFO
) { Reg tmp
= left
; left
= right
; right
= tmp
; }
1282 emit_tab(as
, pi
|PPCF_DOT
, dest
, left
, right
);
1285 #define asm_addov(as, ir) asm_arithov(as, ir, PPCI_ADDO)
1286 #define asm_subov(as, ir) asm_arithov(as, ir, PPCI_SUBFO)
1287 #define asm_mulov(as, ir) asm_arithov(as, ir, PPCI_MULLWO)
1290 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1292 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1293 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1294 PPCIns pi
= PPCI_ADDE
;
1295 if (irref_isk(ir
->op2
)) {
1296 int32_t k
= IR(ir
->op2
)->i
;
1306 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1308 emit_tab(as
, pi
, dest
, left
, right
);
1310 dest
= ra_dest(as
, ir
, RSET_GPR
);
1311 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1312 if (irref_isk(ir
->op2
)) {
1313 int32_t k
= IR(ir
->op2
)->i
;
1315 emit_tai(as
, PPCI_ADDIC
, dest
, left
, k
);
1319 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1320 emit_tab(as
, PPCI_ADDC
, dest
, left
, right
);
1323 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1325 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1326 Reg left
, right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1327 PPCIns pi
= PPCI_SUBFE
;
1328 if (irref_isk(ir
->op1
)) {
1329 int32_t k
= IR(ir
->op1
)->i
;
1339 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1341 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1343 dest
= ra_dest(as
, ir
, RSET_GPR
);
1344 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1345 if (irref_isk(ir
->op1
)) {
1346 int32_t k
= IR(ir
->op1
)->i
;
1348 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1352 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1353 emit_tab(as
, PPCI_SUBFC
, dest
, right
, left
);
1356 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1358 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1359 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1360 emit_tab(as
, PPCI_SUBFZE
, dest
, left
, 0);
1362 dest
= ra_dest(as
, ir
, RSET_GPR
);
1363 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1364 emit_tai(as
, PPCI_SUBFIC
, dest
, left
, 0);
1368 static void asm_bnot(ASMState
*as
, IRIns
*ir
)
1370 Reg dest
, left
, right
;
1371 PPCIns pi
= PPCI_NOR
;
1372 if (as
->flagmcp
== as
->mcp
) {
1377 dest
= ra_dest(as
, ir
, RSET_GPR
);
1378 if (mayfuse(as
, ir
->op1
)) {
1379 IRIns
*irl
= IR(ir
->op1
);
1380 if (irl
->o
== IR_BAND
)
1381 pi
^= (PPCI_NOR
^ PPCI_NAND
);
1382 else if (irl
->o
== IR_BXOR
)
1383 pi
^= (PPCI_NOR
^ PPCI_EQV
);
1384 else if (irl
->o
!= IR_BOR
)
1386 left
= ra_hintalloc(as
, irl
->op1
, dest
, RSET_GPR
);
1387 right
= ra_alloc1(as
, irl
->op2
, rset_exclude(RSET_GPR
, left
));
1390 left
= right
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1392 emit_asb(as
, pi
, dest
, left
, right
);
1395 static void asm_bswap(ASMState
*as
, IRIns
*ir
)
1397 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1399 if (mayfuse(as
, ir
->op1
) && (irx
= IR(ir
->op1
))->o
== IR_XLOAD
&&
1400 ra_noreg(irx
->r
) && (irt_isint(irx
->t
) || irt_isu32(irx
->t
))) {
1401 /* Fuse BSWAP with XLOAD to lwbrx. */
1402 asm_fusexrefx(as
, PPCI_LWBRX
, dest
, irx
->op1
, RSET_GPR
);
1404 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1408 emit_mr(as
, dest
, RID_TMP
);
1410 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 16, 23);
1411 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 0, 7);
1412 emit_rotlwi(as
, tmp
, left
, 8);
1416 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1417 static void asm_fuseandsh(ASMState
*as
, PPCIns pi
, int32_t mask
, IRRef ref
)
1421 if (mayfuse(as
, ref
) && (ir
= IR(ref
), ra_noreg(ir
->r
)) &&
1422 irref_isk(ir
->op2
) && ir
->o
>= IR_BSHL
&& ir
->o
<= IR_BROR
) {
1423 int32_t sh
= (IR(ir
->op2
)->i
& 31);
1426 if ((mask
& ((1u<<sh
)-1))) goto nofuse
;
1429 if ((mask
& ~((~0u)>>sh
))) goto nofuse
;
1437 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1438 *--as
->mcp
= pi
| PPCF_T(left
) | PPCF_B(sh
);
1442 left
= ra_alloc1(as
, ref
, RSET_GPR
);
1443 *--as
->mcp
= pi
| PPCF_T(left
);
1446 static void asm_band(ASMState
*as
, IRIns
*ir
)
1448 Reg dest
, left
, right
;
1449 IRRef lref
= ir
->op1
;
1452 if (as
->flagmcp
== as
->mcp
) {
1457 dest
= ra_dest(as
, ir
, RSET_GPR
);
1458 if (irref_isk(ir
->op2
)) {
1459 int32_t k
= IR(ir
->op2
)->i
;
1461 /* First check for a contiguous bitmask as used by rlwinm. */
1462 uint32_t s1
= lj_ffs((uint32_t)k
);
1463 uint32_t k1
= ((uint32_t)k
>> s1
);
1464 if ((k1
& (k1
+1)) == 0) {
1465 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1466 PPCF_MB(31-lj_fls((uint32_t)k
)) | PPCF_ME(31-s1
),
1471 uint32_t s2
= lj_ffs(~(uint32_t)k
);
1472 uint32_t k2
= (~(uint32_t)k
>> s2
);
1473 if ((k2
& (k2
+1)) == 0) {
1474 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1475 PPCF_MB(32-s2
) | PPCF_ME(30-lj_fls(~(uint32_t)k
)),
1482 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1483 emit_asi(as
, PPCI_ANDIDOT
, dest
, left
, k
);
1485 } else if ((k
& 0xffff) == 0) {
1486 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1487 emit_asi(as
, PPCI_ANDISDOT
, dest
, left
, (k
>> 16));
1492 if (mayfuse(as
, op2
) && IR(op2
)->o
== IR_BNOT
&& ra_noreg(IR(op2
)->r
)) {
1493 dot
^= (PPCI_AND
^ PPCI_ANDC
);
1496 left
= ra_hintalloc(as
, lref
, dest
, RSET_GPR
);
1497 right
= ra_alloc1(as
, op2
, rset_exclude(RSET_GPR
, left
));
1498 emit_asb(as
, PPCI_AND
^ dot
, dest
, left
, right
);
1501 static void asm_bitop(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1503 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1504 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1505 if (irref_isk(ir
->op2
)) {
1506 int32_t k
= IR(ir
->op2
)->i
;
1508 if ((checku16(k
) || (k
& 0xffff) == 0) || (tmp
= dest
, !as
->sectref
)) {
1510 emit_asi(as
, pik
^ (PPCI_ORI
^ PPCI_ORIS
), dest
, tmp
, (k
>> 16));
1511 if ((k
& 0xffff) == 0) return;
1513 emit_asi(as
, pik
, dest
, left
, k
);
1517 /* May fail due to spills/restores above, but simplifies the logic. */
1518 if (as
->flagmcp
== as
->mcp
) {
1523 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1524 emit_asb(as
, pi
, dest
, left
, right
);
1527 #define asm_bor(as, ir) asm_bitop(as, ir, PPCI_OR, PPCI_ORI)
1528 #define asm_bxor(as, ir) asm_bitop(as, ir, PPCI_XOR, PPCI_XORI)
1530 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1534 if (as
->flagmcp
== as
->mcp
) {
1539 dest
= ra_dest(as
, ir
, RSET_GPR
);
1540 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1541 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1542 int32_t shift
= (IR(ir
->op2
)->i
& 31);
1543 if (pik
== 0) /* SLWI */
1544 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, shift
, 0, 31-shift
);
1545 else if (pik
== 1) /* SRWI */
1546 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, (32-shift
)&31, shift
, 31);
1548 emit_asb(as
, pik
|dot
, dest
, left
, shift
);
1550 Reg right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1551 emit_asb(as
, pi
|dot
, dest
, left
, right
);
1555 #define asm_bshl(as, ir) asm_bitshift(as, ir, PPCI_SLW, 0)
1556 #define asm_bshr(as, ir) asm_bitshift(as, ir, PPCI_SRW, 1)
1557 #define asm_bsar(as, ir) asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI)
1558 #define asm_brol(as, ir) \
1559 asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31), \
1560 PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31))
1561 #define asm_bror(as, ir) lua_assert(0)
1563 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1565 if (irt_isnum(ir
->t
)) {
1566 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1568 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1569 right
= (left
>> 8); left
&= 255;
1570 if (tmp
== left
|| tmp
== right
)
1571 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_FPR
,
1572 dest
), left
), right
));
1573 emit_facb(as
, PPCI_FSEL
, dest
, tmp
,
1574 ismax
? left
: right
, ismax
? right
: left
);
1575 emit_fab(as
, PPCI_FSUB
, tmp
, left
, right
);
1577 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1578 Reg tmp1
= RID_TMP
, tmp2
= dest
;
1579 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1580 right
= (left
>> 8); left
&= 255;
1581 if (tmp2
== left
|| tmp2
== right
)
1582 tmp2
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
,
1583 dest
), left
), right
));
1584 emit_tab(as
, PPCI_ADD
, dest
, tmp2
, right
);
1585 emit_asb(as
, ismax
? PPCI_ANDC
: PPCI_AND
, tmp2
, tmp2
, tmp1
);
1586 emit_tab(as
, PPCI_SUBFE
, tmp1
, tmp1
, tmp1
);
1587 emit_tab(as
, PPCI_SUBFC
, tmp2
, tmp2
, tmp1
);
1588 emit_asi(as
, PPCI_XORIS
, tmp2
, right
, 0x8000);
1589 emit_asi(as
, PPCI_XORIS
, tmp1
, left
, 0x8000);
1593 #define asm_min(as, ir) asm_min_max(as, ir, 0)
1594 #define asm_max(as, ir) asm_min_max(as, ir, 1)
1596 /* -- Comparisons --------------------------------------------------------- */
1598 #define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
1599 #define CC_TWO 0x80 /* Check two flags for FP comparison. */
1601 /* Map of comparisons to flags. ORDER IR. */
1602 static const uint8_t asm_compmap
[IR_ABC
+1] = {
1603 /* op int cc FP cc */
1604 /* LT */ CC_GE
+ (CC_GE
<<4),
1605 /* GE */ CC_LT
+ (CC_LE
<<4) + CC_TWO
,
1606 /* LE */ CC_GT
+ (CC_GE
<<4) + CC_TWO
,
1607 /* GT */ CC_LE
+ (CC_LE
<<4),
1608 /* ULT */ CC_GE
+ CC_UNSIGNED
+ (CC_GT
<<4) + CC_TWO
,
1609 /* UGE */ CC_LT
+ CC_UNSIGNED
+ (CC_LT
<<4),
1610 /* ULE */ CC_GT
+ CC_UNSIGNED
+ (CC_GT
<<4),
1611 /* UGT */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
,
1612 /* EQ */ CC_NE
+ (CC_NE
<<4),
1613 /* NE */ CC_EQ
+ (CC_EQ
<<4),
1614 /* ABC */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
/* Same as UGT. */
1617 static void asm_intcomp_(ASMState
*as
, IRRef lref
, IRRef rref
, Reg cr
, PPCCC cc
)
1619 Reg right
, left
= ra_alloc1(as
, lref
, RSET_GPR
);
1620 if (irref_isk(rref
)) {
1621 int32_t k
= IR(rref
)->i
;
1622 if ((cc
& CC_UNSIGNED
) == 0) { /* Signed comparison with constant. */
1624 emit_tai(as
, PPCI_CMPWI
, cr
, left
, k
);
1625 /* Signed comparison with zero and referencing previous ins? */
1626 if (k
== 0 && lref
== as
->curins
-1)
1627 as
->flagmcp
= as
->mcp
; /* Allow elimination of the compare. */
1629 } else if ((cc
& 3) == (CC_EQ
& 3)) { /* Use CMPLWI for EQ or NE. */
1631 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1633 } else if (!as
->sectref
&& ra_noreg(IR(rref
)->r
)) {
1634 emit_tai(as
, PPCI_CMPLWI
, cr
, RID_TMP
, k
);
1635 emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, (k
>> 16));
1639 } else { /* Unsigned comparison with constant. */
1641 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1646 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, left
));
1647 emit_tab(as
, (cc
& CC_UNSIGNED
) ? PPCI_CMPLW
: PPCI_CMPW
, cr
, left
, right
);
1650 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1652 PPCCC cc
= asm_compmap
[ir
->o
];
1653 if (irt_isnum(ir
->t
)) {
1654 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1655 right
= (left
>> 8); left
&= 255;
1656 asm_guardcc(as
, (cc
>> 4));
1658 emit_tab(as
, PPCI_CROR
, ((cc
>>4)&3), ((cc
>>4)&3), (CC_EQ
&3));
1659 emit_fab(as
, PPCI_FCMPU
, 0, left
, right
);
1661 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1662 if (irref_isk(lref
) && !irref_isk(rref
)) {
1663 /* Swap constants to the right (only for ABC). */
1664 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1665 if ((cc
& 2) == 0) cc
^= 1; /* LT <-> GT, LE <-> GE */
1667 asm_guardcc(as
, cc
);
1668 asm_intcomp_(as
, lref
, rref
, 0, cc
);
1672 #define asm_equal(as, ir) asm_comp(as, ir)
1675 /* 64 bit integer comparisons. */
1676 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1678 PPCCC cc
= asm_compmap
[(ir
-1)->o
];
1679 if ((cc
&3) == (CC_EQ
&3)) {
1680 asm_guardcc(as
, cc
);
1681 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CROR
,
1682 (CC_EQ
&3), (CC_EQ
&3), 4+(CC_EQ
&3));
1684 asm_guardcc(as
, CC_EQ
);
1685 emit_tab(as
, PPCI_CROR
, (CC_EQ
&3), (CC_EQ
&3), ((cc
^~(cc
>>2))&1));
1686 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CRANDC
,
1687 (CC_EQ
&3), (CC_EQ
&3), 4+(cc
&3));
1689 /* Loword comparison sets cr1 and is unsigned, except for equality. */
1690 asm_intcomp_(as
, (ir
-1)->op1
, (ir
-1)->op2
, 4,
1691 cc
| ((cc
&3) == (CC_EQ
&3) ? 0 : CC_UNSIGNED
));
1692 /* Hiword comparison sets cr0. */
1693 asm_intcomp_(as
, ir
->op1
, ir
->op2
, 0, cc
);
1694 as
->flagmcp
= NULL
; /* Doesn't work here. */
1698 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1700 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1701 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1704 /* HIOP is marked as a store because it needs its own DCE logic. */
1705 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1706 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1707 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1708 as
->curins
--; /* Always skip the CONV. */
1712 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1713 as
->curins
--; /* Always skip the loword comparison. */
1716 } else if ((ir
-1)->o
== IR_XSTORE
) {
1717 as
->curins
--; /* Handle both stores here. */
1718 if ((ir
-1)->r
!= RID_SINK
) {
1719 asm_xstore_(as
, ir
, 0);
1720 asm_xstore_(as
, ir
-1, 4);
1724 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1725 switch ((ir
-1)->o
) {
1726 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1727 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1728 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1732 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1735 /* Nothing to do here. Handled by lo op itself. */
1737 default: lua_assert(0); break;
1740 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1744 /* -- Profiling ----------------------------------------------------------- */
1746 static void asm_prof(ASMState
*as
, IRIns
*ir
)
1749 asm_guardcc(as
, CC_NE
);
1750 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, HOOK_PROFILE
);
1751 emit_lsglptr(as
, PPCI_LBZ
, RID_TMP
,
1752 (int32_t)offsetof(global_State
, hookmask
));
1755 /* -- Stack handling ------------------------------------------------------ */
1757 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1758 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1759 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1761 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1762 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1763 rset_clear(allow
, pbase
);
1764 tmp
= allow
? rset_pickbot(allow
) :
1765 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1766 emit_condbranch(as
, PPCI_BC
, CC_LT
, asm_exitstub_addr(as
, exitno
));
1767 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1768 emit_tai(as
, PPCI_LWZ
, tmp
, RID_SP
, SPOFS_TMPW
);
1770 ra_modified(as
, tmp
);
1771 emit_ai(as
, PPCI_CMPLWI
, RID_TMP
, (int32_t)(8*topslot
));
1772 emit_tab(as
, PPCI_SUBF
, RID_TMP
, pbase
, tmp
);
1773 emit_tai(as
, PPCI_LWZ
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1774 if (pbase
== RID_TMP
)
1775 emit_getgl(as
, RID_TMP
, jit_base
);
1776 emit_getgl(as
, tmp
, cur_L
);
1777 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1778 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPW
);
1781 /* Restore Lua stack from on-trace state. */
1782 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1784 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1785 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1786 MSize n
, nent
= snap
->nent
;
1787 /* Store the value of all modified slots to the Lua stack. */
1788 for (n
= 0; n
< nent
; n
++) {
1789 SnapEntry sn
= map
[n
];
1790 BCReg s
= snap_slot(sn
);
1791 int32_t ofs
= 8*((int32_t)s
-1);
1792 IRRef ref
= snap_ref(sn
);
1793 IRIns
*ir
= IR(ref
);
1794 if ((sn
& SNAP_NORESTORE
))
1796 if (irt_isnum(ir
->t
)) {
1797 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1798 emit_fai(as
, PPCI_STFD
, src
, RID_BASE
, ofs
);
1801 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1802 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1803 if (!irt_ispri(ir
->t
)) {
1804 Reg src
= ra_alloc1(as
, ref
, allow
);
1805 rset_clear(allow
, src
);
1806 emit_tai(as
, PPCI_STW
, src
, RID_BASE
, ofs
+4);
1808 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1809 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1810 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1812 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1814 emit_tai(as
, PPCI_STW
, type
, RID_BASE
, ofs
);
1818 lua_assert(map
+ nent
== flinks
);
1821 /* -- GC handling --------------------------------------------------------- */
1823 /* Check GC threshold and do one or more GC steps. */
1824 static void asm_gc_check(ASMState
*as
)
1826 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1830 ra_evictset(as
, RSET_SCRATCH
);
1831 l_end
= emit_label(as
);
1832 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1833 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
1834 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0);
1835 args
[0] = ASMREF_TMP1
; /* global_State *g */
1836 args
[1] = ASMREF_TMP2
; /* MSize steps */
1837 asm_gencall(as
, ci
, args
);
1838 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1839 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1840 emit_loadi(as
, tmp
, as
->gcsteps
);
1841 /* Jump around GC step if GC total < GC threshold. */
1842 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_LT
, l_end
);
1843 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tmp
);
1844 emit_getgl(as
, tmp
, gc
.threshold
);
1845 emit_getgl(as
, RID_TMP
, gc
.total
);
1850 /* -- Loop handling ------------------------------------------------------- */
1852 /* Fixup the loop branch. */
1853 static void asm_loop_fixup(ASMState
*as
)
1855 MCode
*p
= as
->mctop
;
1856 MCode
*target
= as
->mcp
;
1857 if (as
->loopinv
) { /* Inverted loop branch? */
1858 /* asm_guardcc already inverted the cond branch and patched the final b. */
1859 p
[-2] = (p
[-2] & (0xffff0000u
& ~PPCF_Y
)) | (((target
-p
+2) & 0x3fffu
) << 2);
1861 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1865 /* -- Head of trace ------------------------------------------------------- */
1867 /* Coalesce BASE register for a root trace. */
1868 static void asm_head_root_base(ASMState
*as
)
1870 IRIns
*ir
= IR(REF_BASE
);
1874 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1875 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1877 emit_mr(as
, r
, RID_BASE
);
1881 /* Coalesce BASE register for a side trace. */
1882 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1884 IRIns
*ir
= IR(REF_BASE
);
1888 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1889 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1891 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1892 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1893 rset_clear(allow
, irp
->r
);
1894 emit_mr(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1896 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1902 /* -- Tail of trace ------------------------------------------------------- */
1904 /* Fixup the tail code. */
1905 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1907 MCode
*p
= as
->mctop
;
1909 int32_t spadj
= as
->T
->spadjust
;
1915 /* Patch stack adjustment. */
1916 lua_assert(checki16(CFRAME_SIZE
+spadj
));
1917 p
[-3] = PPCI_ADDI
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | (CFRAME_SIZE
+spadj
);
1918 p
[-2] = PPCI_STWU
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | spadj
;
1920 /* Patch exit branch. */
1921 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1922 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1925 /* Prepare tail of code. */
1926 static void asm_tail_prep(ASMState
*as
)
1928 MCode
*p
= as
->mctop
- 1; /* Leave room for exit branch. */
1930 as
->invmcp
= as
->mcp
= p
;
1932 as
->mcp
= p
-2; /* Leave room for stack pointer adjustment. */
1937 /* -- Trace setup --------------------------------------------------------- */
1939 /* Ensure there are enough stack slots for call arguments. */
1940 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
1942 IRRef args
[CCI_NARGS_MAX
*2];
1943 uint32_t i
, nargs
= CCI_XNARGS(ci
);
1944 int nslots
= 2, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
1945 asm_collectargs(as
, ir
, ci
, args
);
1946 for (i
= 0; i
< nargs
; i
++)
1947 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
1948 if (nfpr
> 0) nfpr
--; else nslots
= (nslots
+3) & ~1;
1950 if (ngpr
> 0) ngpr
--; else nslots
++;
1952 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
1953 as
->evenspill
= nslots
;
1954 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
1957 static void asm_setup_target(ASMState
*as
)
1959 asm_exitstub_setup(as
, as
->T
->nsnap
+ (as
->parent
? 1 : 0));
1962 /* -- Trace patching ------------------------------------------------------ */
1964 /* Patch exit jumps of existing machine code to a new target. */
1965 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
1967 MCode
*p
= T
->mcode
;
1968 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
1969 MCode
*px
= exitstub_trace_addr(T
, exitno
);
1970 MCode
*cstart
= NULL
;
1971 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
1973 for (; p
< pe
; p
++) {
1974 /* Look for exitstub branch, try to replace with branch to target. */
1976 if ((ins
& 0xfc000000u
) == 0x40000000u
&&
1977 ((ins
^ ((char *)px
-(char *)p
)) & 0xffffu
) == 0) {
1978 ptrdiff_t delta
= (char *)target
- (char *)p
;
1979 if (((ins
>> 16) & 3) == (CC_SO
&3)) {
1980 clearso
= sizeof(MCode
);
1981 delta
-= sizeof(MCode
);
1983 /* Many, but not all short-range branches can be patched directly. */
1984 if (((delta
+ 0x8000) >> 16) == 0) {
1985 *p
= (ins
& 0xffdf0000u
) | ((uint32_t)delta
& 0xffffu
) |
1986 ((delta
& 0x8000) * (PPCF_Y
/0x8000));
1987 if (!cstart
) cstart
= p
;
1989 } else if ((ins
& 0xfc000000u
) == PPCI_B
&&
1990 ((ins
^ ((char *)px
-(char *)p
)) & 0x03ffffffu
) == 0) {
1991 ptrdiff_t delta
= (char *)target
- (char *)p
;
1992 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
1993 *p
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
1994 if (!cstart
) cstart
= p
;
1997 { /* Always patch long-range branch in exit stub itself. */
1998 ptrdiff_t delta
= (char *)target
- (char *)px
- clearso
;
1999 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2000 *px
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2002 if (!cstart
) cstart
= px
;
2003 lj_mcode_sync(cstart
, px
+1);
2004 if (clearso
) { /* Extend the current trace. Ugly workaround. */
2005 MCode
*pp
= J
->cur
.mcode
;
2006 J
->cur
.szmcode
+= sizeof(MCode
);
2007 *--pp
= PPCI_MCRXR
; /* Clear SO flag. */
2009 lj_mcode_sync(pp
, pp
+1);
2011 lj_mcode_patch(J
, mcarea
, 1);