2 ** PPC IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate two source registers for three-operand instructions. */
22 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
24 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
25 Reg left
= irl
->r
, right
= irr
->r
;
26 if (ra_hasreg(left
)) {
29 right
= ra_allocref(as
, ir
->op2
, rset_exclude(allow
, left
));
32 } else if (ra_hasreg(right
)) {
34 left
= ra_allocref(as
, ir
->op1
, rset_exclude(allow
, right
));
35 } else if (ra_hashint(right
)) {
36 right
= ra_allocref(as
, ir
->op2
, allow
);
37 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(allow
, right
));
39 left
= ra_allocref(as
, ir
->op1
, allow
);
40 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(allow
, left
));
42 return left
| (right
<< 8);
45 /* -- Guard handling ------------------------------------------------------ */
47 /* Setup exit stubs after the end of each trace. */
48 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
51 MCode
*mxp
= as
->mctop
;
52 if (mxp
- (nexits
+ 3 + MCLIM_REDZONE
) < as
->mclim
)
54 /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
55 for (i
= nexits
-1; (int32_t)i
>= 0; i
--)
56 *--mxp
= PPCI_BL
|(((-3-i
)&0x00ffffffu
)<<2);
57 *--mxp
= PPCI_LI
|PPCF_T(RID_TMP
)|as
->T
->traceno
; /* Read by exit handler. */
59 *mxp
= PPCI_BL
|((((MCode
*)(void *)lj_vm_exit_handler
-mxp
)&0x00ffffffu
)<<2);
60 *--mxp
= PPCI_MFLR
|PPCF_T(RID_TMP
);
64 static MCode
*asm_exitstub_addr(ASMState
*as
, ExitNo exitno
)
66 /* Keep this in-sync with exitstub_trace_addr(). */
67 return as
->mctop
+ exitno
+ 3;
70 /* Emit conditional branch to exit for guard. */
71 static void asm_guardcc(ASMState
*as
, PPCCC cc
)
73 MCode
*target
= asm_exitstub_addr(as
, as
->snapno
);
75 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
77 *p
= PPCI_B
| (((target
-p
) & 0x00ffffffu
) << 2);
78 emit_condbranch(as
, PPCI_BC
, cc
^4, p
);
81 emit_condbranch(as
, PPCI_BC
, cc
, target
);
84 /* -- Operand fusion ------------------------------------------------------ */
86 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
87 #define CONFLICT_SEARCH_LIM 31
89 /* Check if there's no conflicting instruction between curins and ref. */
90 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
94 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
95 return 0; /* Give up, ref is too far away. */
97 if (ir
[i
].o
== conflict
)
98 return 0; /* Conflict found. */
99 return 1; /* Ok, no conflict. */
102 /* Fuse the array base of colocated arrays. */
103 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
106 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
107 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
108 return (int32_t)sizeof(GCtab
);
112 /* Indicates load/store indexed is ok. */
113 #define AHUREF_LSX ((int32_t)0x80000000)
115 /* Fuse array/hash/upvalue reference into register+offset operand. */
116 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
119 if (ra_noreg(ir
->r
)) {
120 if (ir
->o
== IR_AREF
) {
121 if (mayfuse(as
, ref
)) {
122 if (irref_isk(ir
->op2
)) {
123 IRRef tab
= IR(ir
->op1
)->op1
;
124 int32_t ofs
= asm_fuseabase(as
, tab
);
125 IRRef refa
= ofs
? tab
: ir
->op1
;
126 ofs
+= 8*IR(ir
->op2
)->i
;
129 return ra_alloc1(as
, refa
, allow
);
132 if (*ofsp
== AHUREF_LSX
) {
133 Reg base
= ra_alloc1(as
, ir
->op1
, allow
);
134 Reg idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
135 return base
| (idx
<< 8);
138 } else if (ir
->o
== IR_HREFK
) {
139 if (mayfuse(as
, ref
)) {
140 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
143 return ra_alloc1(as
, ir
->op1
, allow
);
146 } else if (ir
->o
== IR_UREFC
) {
147 if (irref_isk(ir
->op1
)) {
148 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
149 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
150 int32_t jgl
= (intptr_t)J2G(as
->J
);
151 if ((uint32_t)(ofs
-jgl
) < 65536) {
152 *ofsp
= ofs
-jgl
-32768;
155 *ofsp
= (int16_t)ofs
;
156 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
162 return ra_alloc1(as
, ref
, allow
);
165 /* Fuse XLOAD/XSTORE reference into load/store operand. */
166 static void asm_fusexref(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
167 RegSet allow
, int32_t ofs
)
171 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
172 if (ir
->o
== IR_ADD
) {
174 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
177 } else if (ofs
== 0) {
178 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
179 right
= (left
>> 8); left
&= 255;
180 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
183 } else if (ir
->o
== IR_STRREF
) {
184 lua_assert(ofs
== 0);
185 ofs
= (int32_t)sizeof(GCstr
);
186 if (irref_isk(ir
->op2
)) {
187 ofs
+= IR(ir
->op2
)->i
;
189 } else if (irref_isk(ir
->op1
)) {
190 ofs
+= IR(ir
->op1
)->i
;
193 /* NYI: Fuse ADD with constant. */
194 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, allow
);
195 right
= (left
>> 8); left
&= 255;
196 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(allow
, left
), right
));
197 emit_fai(as
, pi
, rt
, tmp
, ofs
);
198 emit_tab(as
, PPCI_ADD
, tmp
, left
, right
);
201 if (!checki16(ofs
)) {
202 Reg left
= ra_alloc1(as
, ref
, allow
);
203 Reg right
= ra_allock(as
, ofs
, rset_exclude(allow
, left
));
204 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
209 base
= ra_alloc1(as
, ref
, allow
);
210 emit_fai(as
, pi
, rt
, base
, ofs
);
213 /* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
214 static void asm_fusexrefx(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
217 IRIns
*ira
= IR(ref
);
219 if (canfuse(as
, ira
) && ira
->o
== IR_ADD
&& ra_noreg(ira
->r
)) {
220 left
= ra_alloc2(as
, ira
, allow
);
221 right
= (left
>> 8); left
&= 255;
223 right
= ra_alloc1(as
, ref
, allow
);
226 emit_tab(as
, pi
, rt
, left
, right
);
229 /* Fuse to multiply-add/sub instruction. */
230 static int asm_fusemadd(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pir
)
232 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
235 ((mayfuse(as
, lref
) && (irm
= IR(lref
), irm
->o
== IR_MUL
) &&
237 (mayfuse(as
, rref
) && (irm
= IR(rref
), irm
->o
== IR_MUL
) &&
238 (rref
= lref
, pi
= pir
, ra_noreg(irm
->r
))))) {
239 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
240 Reg add
= ra_alloc1(as
, rref
, RSET_FPR
);
241 Reg right
, left
= ra_alloc2(as
, irm
, rset_exclude(RSET_FPR
, add
));
242 right
= (left
>> 8); left
&= 255;
243 emit_facb(as
, pi
, dest
, left
, right
, add
);
249 /* -- Calls --------------------------------------------------------------- */
251 /* Generate a call to a C function. */
252 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
254 uint32_t n
, nargs
= CCI_XNARGS(ci
);
256 Reg gpr
= REGARG_FIRSTGPR
, fpr
= REGARG_FIRSTFPR
;
257 if ((void *)ci
->func
)
258 emit_call(as
, (void *)ci
->func
);
259 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
263 if (irt_isfp(ir
->t
)) {
264 if (fpr
<= REGARG_LASTFPR
) {
265 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
266 ra_leftov(as
, fpr
, ref
);
269 Reg r
= ra_alloc1(as
, ref
, RSET_FPR
);
270 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
271 emit_spstore(as
, ir
, r
, ofs
);
272 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
275 if (gpr
<= REGARG_LASTGPR
) {
276 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
277 ra_leftov(as
, gpr
, ref
);
280 Reg r
= ra_alloc1(as
, ref
, RSET_GPR
);
281 emit_spstore(as
, ir
, r
, ofs
);
286 if (gpr
<= REGARG_LASTGPR
)
293 if ((ci
->flags
& CCI_VARARG
)) /* Vararg calls need to know about FPR use. */
294 emit_tab(as
, fpr
== REGARG_FIRSTFPR
? PPCI_CRXOR
: PPCI_CREQV
, 6, 6, 6);
297 /* Setup result reg/sp for call. Evict scratch regs. */
298 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
300 RegSet drop
= RSET_SCRATCH
;
301 int hiop
= ((ir
+1)->o
== IR_HIOP
);
302 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
304 if (ra_hasreg(ir
->r
))
305 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
306 if (hiop
&& ra_hasreg((ir
+1)->r
))
307 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
308 ra_evictset(as
, drop
); /* Evictions must be performed first. */
310 lua_assert(!irt_ispri(ir
->t
));
311 if (irt_isfp(ir
->t
)) {
312 if ((ci
->flags
& CCI_CASTU64
)) {
313 /* Use spill slot or temp slots. */
314 int32_t ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
316 if (ra_hasreg(dest
)) {
318 ra_modified(as
, dest
);
319 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, ofs
);
321 emit_tai(as
, PPCI_STW
, RID_RETHI
, RID_SP
, ofs
);
322 emit_tai(as
, PPCI_STW
, RID_RETLO
, RID_SP
, ofs
+4);
324 ra_destreg(as
, ir
, RID_FPRET
);
331 ra_destreg(as
, ir
, RID_RET
);
336 static void asm_callx(ASMState
*as
, IRIns
*ir
)
338 IRRef args
[CCI_NARGS_MAX
*2];
342 ci
.flags
= asm_callx_flags(as
, ir
);
343 asm_collectargs(as
, ir
, &ci
, args
);
344 asm_setupresult(as
, ir
, &ci
);
345 func
= ir
->op2
; irf
= IR(func
);
346 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
347 if (irref_isk(func
)) { /* Call to constant address. */
348 ci
.func
= (ASMFunction
)(void *)(intptr_t)(irf
->i
);
349 } else { /* Need a non-argument register for indirect calls. */
350 RegSet allow
= RSET_GPR
& ~RSET_RANGE(RID_R0
, REGARG_LASTGPR
+1);
351 Reg freg
= ra_alloc1(as
, func
, allow
);
352 *--as
->mcp
= PPCI_BCTRL
;
353 *--as
->mcp
= PPCI_MTCTR
| PPCF_T(freg
);
354 ci
.func
= (ASMFunction
)(void *)0;
356 asm_gencall(as
, &ci
, args
);
359 /* -- Returns ------------------------------------------------------------- */
361 /* Return to lower frame. Guard that it goes to the right spot. */
362 static void asm_retf(ASMState
*as
, IRIns
*ir
)
364 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
365 void *pc
= ir_kptr(IR(ir
->op2
));
366 int32_t delta
= 1+LJ_FR2
+bc_a(*((const BCIns
*)pc
- 1));
367 as
->topslot
-= (BCReg
)delta
;
368 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
369 irt_setmark(IR(REF_BASE
)->t
); /* Children must not coalesce with BASE reg. */
370 emit_setgl(as
, base
, jit_base
);
371 emit_addptr(as
, base
, -8*delta
);
372 asm_guardcc(as
, CC_NE
);
373 emit_ab(as
, PPCI_CMPW
, RID_TMP
,
374 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
375 emit_tai(as
, PPCI_LWZ
, RID_TMP
, base
, -8);
378 /* -- Type conversions ---------------------------------------------------- */
380 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
382 RegSet allow
= RSET_FPR
;
383 Reg tmp
= ra_scratch(as
, rset_clear(allow
, left
));
384 Reg fbias
= ra_scratch(as
, rset_clear(allow
, tmp
));
385 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
386 Reg hibias
= ra_allock(as
, 0x43300000, rset_exclude(RSET_GPR
, dest
));
387 asm_guardcc(as
, CC_NE
);
388 emit_fab(as
, PPCI_FCMPU
, 0, tmp
, left
);
389 emit_fab(as
, PPCI_FSUB
, tmp
, tmp
, fbias
);
390 emit_fai(as
, PPCI_LFD
, tmp
, RID_SP
, SPOFS_TMP
);
391 emit_tai(as
, PPCI_STW
, RID_TMP
, RID_SP
, SPOFS_TMPLO
);
392 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
393 emit_asi(as
, PPCI_XORIS
, RID_TMP
, dest
, 0x8000);
394 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
395 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
396 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
398 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
399 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
402 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
404 RegSet allow
= RSET_FPR
;
405 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
406 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
407 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
408 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
409 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
410 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
411 emit_fab(as
, PPCI_FADD
, tmp
, left
, right
);
414 static void asm_conv(ASMState
*as
, IRIns
*ir
)
416 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
417 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
418 IRRef lref
= ir
->op1
;
419 lua_assert(irt_type(ir
->t
) != st
);
420 lua_assert(!(irt_isint64(ir
->t
) ||
421 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
422 if (irt_isfp(ir
->t
)) {
423 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
424 if (stfp
) { /* FP to FP conversion. */
425 if (st
== IRT_NUM
) /* double -> float conversion. */
426 emit_fb(as
, PPCI_FRSP
, dest
, ra_alloc1(as
, lref
, RSET_FPR
));
427 else /* float -> double conversion is a no-op on PPC. */
428 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
429 } else { /* Integer to FP conversion. */
430 /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
431 /* IRT_U32: Bias with 2^52, subtract 2^52. */
432 RegSet allow
= RSET_GPR
;
433 Reg left
= ra_alloc1(as
, lref
, allow
);
434 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, left
));
435 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
437 if (irt_isfloat(ir
->t
)) emit_fb(as
, PPCI_FRSP
, dest
, dest
);
438 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
439 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
440 kbias
= (const float *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000));
441 if (st
== IRT_U32
) kbias
++;
442 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31), (void *)kbias
,
443 rset_clear(allow
, hibias
));
444 emit_tai(as
, PPCI_STW
, st
== IRT_U32
? left
: RID_TMP
,
445 RID_SP
, SPOFS_TMPLO
);
446 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
447 if (st
!= IRT_U32
) emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, 0x8000);
449 } else if (stfp
) { /* FP to integer conversion. */
450 if (irt_isguard(ir
->t
)) {
451 /* Checked conversions are only supported from number to int. */
452 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
453 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
455 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
456 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
457 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
458 if (irt_isu32(ir
->t
)) {
459 /* Convert both x and x-2^31 to int and merge results. */
460 Reg tmpi
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
461 emit_asb(as
, PPCI_OR
, dest
, dest
, tmpi
); /* Select with mask idiom. */
462 emit_asb(as
, PPCI_AND
, tmpi
, tmpi
, RID_TMP
);
463 emit_asb(as
, PPCI_ANDC
, dest
, dest
, RID_TMP
);
464 emit_tai(as
, PPCI_LWZ
, tmpi
, RID_SP
, SPOFS_TMPLO
); /* tmp = (int)(x) */
465 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, 0x8000); /* dest += 2^31 */
466 emit_asb(as
, PPCI_SRAWI
, RID_TMP
, dest
, 31); /* mask = -(dest < 0) */
467 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
468 emit_tai(as
, PPCI_LWZ
, dest
,
469 RID_SP
, SPOFS_TMPLO
); /* dest = (int)(x-2^31) */
470 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
471 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
472 emit_fb(as
, PPCI_FCTIWZ
, tmp
, tmp
);
473 emit_fab(as
, PPCI_FSUB
, tmp
, left
, tmp
);
474 emit_lsptr(as
, PPCI_LFS
, (tmp
& 31),
475 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,00000000)),
478 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
479 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
480 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
484 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
485 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
486 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
487 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
488 if ((ir
->op2
& IRCONV_SEXT
))
489 emit_as(as
, st
== IRT_I8
? PPCI_EXTSB
: PPCI_EXTSH
, dest
, left
);
491 emit_rot(as
, PPCI_RLWINM
, dest
, left
, 0, st
== IRT_U8
? 24 : 16, 31);
492 } else { /* 32/64 bit integer conversions. */
493 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
494 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
499 static void asm_strto(ASMState
*as
, IRIns
*ir
)
501 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
504 RegSet drop
= RSET_SCRATCH
;
505 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
506 ra_evictset(as
, drop
);
507 asm_guardcc(as
, CC_EQ
);
508 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0); /* Test return status. */
509 args
[0] = ir
->op1
; /* GCstr *str */
510 args
[1] = ASMREF_TMP1
; /* TValue *n */
511 asm_gencall(as
, ci
, args
);
512 /* Store the result to the spill slot or temp slots. */
513 ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
514 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_SP
, ofs
);
517 /* -- Memory references --------------------------------------------------- */
519 /* Get pointer to TValue. */
520 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
523 if (irt_isnum(ir
->t
)) {
524 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
525 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
526 else /* Otherwise force a spill and use the spill slot. */
527 emit_tai(as
, PPCI_ADDI
, dest
, RID_SP
, ra_spill(as
, ir
));
529 /* Otherwise use g->tmptv to hold the TValue. */
530 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
532 emit_tai(as
, PPCI_ADDI
, dest
, RID_JGL
, (int32_t)offsetof(global_State
, tmptv
)-32768);
533 if (!irt_ispri(ir
->t
)) {
534 Reg src
= ra_alloc1(as
, ref
, allow
);
535 emit_setgl(as
, src
, tmptv
.gcr
);
537 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
538 emit_setgl(as
, type
, tmptv
.it
);
542 static void asm_aref(ASMState
*as
, IRIns
*ir
)
544 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
546 if (irref_isk(ir
->op2
)) {
547 IRRef tab
= IR(ir
->op1
)->op1
;
548 int32_t ofs
= asm_fuseabase(as
, tab
);
549 IRRef refa
= ofs
? tab
: ir
->op1
;
550 ofs
+= 8*IR(ir
->op2
)->i
;
552 base
= ra_alloc1(as
, refa
, RSET_GPR
);
553 emit_tai(as
, PPCI_ADDI
, dest
, base
, ofs
);
557 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
558 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
559 emit_tab(as
, PPCI_ADD
, dest
, RID_TMP
, base
);
560 emit_slwi(as
, RID_TMP
, idx
, 3);
563 /* Inlined hash lookup. Specialized for key type and for const keys.
564 ** The equivalent C code is:
565 ** Node *n = hashkey(t, key);
567 ** if (lj_obj_equal(&n->key, key)) return &n->val;
568 ** } while ((n = nextnode(n)));
571 static void asm_href(ASMState
*as
, IRIns
*ir
, IROp merge
)
573 RegSet allow
= RSET_GPR
;
574 int destused
= ra_used(ir
);
575 Reg dest
= ra_dest(as
, ir
, allow
);
576 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
577 Reg key
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
578 Reg tisnum
= RID_NONE
, tmpnum
= RID_NONE
;
579 IRRef refkey
= ir
->op2
;
580 IRIns
*irkey
= IR(refkey
);
581 IRType1 kt
= irkey
->t
;
583 MCLabel l_end
, l_loop
, l_next
;
585 rset_clear(allow
, tab
);
587 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
588 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
589 tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
590 rset_clear(allow
, tisnum
);
591 } else if (!irt_ispri(kt
)) {
592 key
= ra_alloc1(as
, refkey
, allow
);
593 rset_clear(allow
, key
);
595 tmp2
= ra_scratch(as
, allow
);
596 rset_clear(allow
, tmp2
);
598 /* Key not found in chain: jump to exit (if merged) or load niltv. */
599 l_end
= emit_label(as
);
602 asm_guardcc(as
, CC_EQ
);
604 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
606 /* Follow hash chain until the end. */
608 emit_ai(as
, PPCI_CMPWI
, dest
, 0);
609 emit_tai(as
, PPCI_LWZ
, dest
, dest
, (int32_t)offsetof(Node
, next
));
610 l_next
= emit_label(as
);
612 /* Type and value comparison. */
614 asm_guardcc(as
, CC_EQ
);
616 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
618 emit_fab(as
, PPCI_FCMPU
, 0, tmpnum
, key
);
619 emit_condbranch(as
, PPCI_BC
, CC_GE
, l_next
);
620 emit_ab(as
, PPCI_CMPLW
, tmp1
, tisnum
);
621 emit_fai(as
, PPCI_LFD
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
623 if (!irt_ispri(kt
)) {
624 emit_ab(as
, PPCI_CMPW
, tmp2
, key
);
625 emit_condbranch(as
, PPCI_BC
, CC_NE
, l_next
);
627 emit_ai(as
, PPCI_CMPWI
, tmp1
, irt_toitype(irkey
->t
));
629 emit_tai(as
, PPCI_LWZ
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
631 emit_tai(as
, PPCI_LWZ
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
632 *l_loop
= PPCI_BC
| PPCF_Y
| PPCF_CC(CC_NE
) |
633 (((char *)as
->mcp
-(char *)l_loop
) & 0xffffu
);
635 /* Load main position relative to tab->node into dest. */
636 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
638 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
641 if (irref_isk(refkey
))
642 tmphash
= ra_allock(as
, khash
, allow
);
643 emit_tab(as
, PPCI_ADD
, dest
, dest
, tmp1
);
644 emit_tai(as
, PPCI_MULLI
, tmp1
, tmp1
, sizeof(Node
));
645 emit_asb(as
, PPCI_AND
, tmp1
, tmp2
, tmphash
);
646 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
647 emit_tai(as
, PPCI_LWZ
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
648 if (irref_isk(refkey
)) {
650 } else if (irt_isstr(kt
)) {
651 emit_tai(as
, PPCI_LWZ
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
652 } else { /* Must match with hash*() in lj_tab.c. */
653 emit_tab(as
, PPCI_SUBF
, tmp1
, tmp2
, tmp1
);
654 emit_rotlwi(as
, tmp2
, tmp2
, HASH_ROT3
);
655 emit_asb(as
, PPCI_XOR
, tmp1
, tmp1
, tmp2
);
656 emit_rotlwi(as
, tmp1
, tmp1
, (HASH_ROT2
+HASH_ROT1
)&31);
657 emit_tab(as
, PPCI_SUBF
, tmp2
, dest
, tmp2
);
659 int32_t ofs
= ra_spill(as
, irkey
);
660 emit_asb(as
, PPCI_XOR
, tmp2
, tmp2
, tmp1
);
661 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
662 emit_tab(as
, PPCI_ADD
, tmp1
, tmp1
, tmp1
);
663 emit_tai(as
, PPCI_LWZ
, tmp2
, RID_SP
, ofs
+4);
664 emit_tai(as
, PPCI_LWZ
, tmp1
, RID_SP
, ofs
);
666 emit_asb(as
, PPCI_XOR
, tmp2
, key
, tmp1
);
667 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
668 emit_tai(as
, PPCI_ADDI
, tmp1
, tmp2
, HASH_BIAS
);
669 emit_tai(as
, PPCI_ADDIS
, tmp2
, key
, (HASH_BIAS
+ 32768)>>16);
675 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
677 IRIns
*kslot
= IR(ir
->op2
);
678 IRIns
*irkey
= IR(kslot
->op1
);
679 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
680 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
681 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
682 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
683 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
684 RegSet allow
= rset_exclude(RSET_GPR
, node
);
685 lua_assert(ofs
% sizeof(Node
) == 0);
688 rset_clear(allow
, dest
);
689 kofs
= (int32_t)offsetof(Node
, key
);
690 } else if (ra_hasreg(dest
)) {
691 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
693 asm_guardcc(as
, CC_NE
);
694 if (!irt_ispri(irkey
->t
)) {
695 key
= ra_scratch(as
, allow
);
696 rset_clear(allow
, key
);
698 rset_clear(allow
, type
);
699 if (irt_isnum(irkey
->t
)) {
700 emit_cmpi(as
, key
, (int32_t)ir_knum(irkey
)->u32
.lo
);
701 asm_guardcc(as
, CC_NE
);
702 emit_cmpi(as
, type
, (int32_t)ir_knum(irkey
)->u32
.hi
);
704 if (ra_hasreg(key
)) {
705 emit_cmpi(as
, key
, irkey
->i
); /* May use RID_TMP, i.e. type. */
706 asm_guardcc(as
, CC_NE
);
708 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(irkey
->t
));
710 if (ra_hasreg(key
)) emit_tai(as
, PPCI_LWZ
, key
, idx
, kofs
+4);
711 emit_tai(as
, PPCI_LWZ
, type
, idx
, kofs
);
713 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (ofs
+ 32768) >> 16);
714 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
718 static void asm_uref(ASMState
*as
, IRIns
*ir
)
720 /* NYI: Check that UREFO is still open and not aliasing a slot. */
721 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
722 if (irref_isk(ir
->op1
)) {
723 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
724 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
725 emit_lsptr(as
, PPCI_LWZ
, dest
, v
, RSET_GPR
);
727 Reg uv
= ra_scratch(as
, RSET_GPR
);
728 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
729 if (ir
->o
== IR_UREFC
) {
730 asm_guardcc(as
, CC_NE
);
731 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, 1);
732 emit_tai(as
, PPCI_ADDI
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
733 emit_tai(as
, PPCI_LBZ
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
735 emit_tai(as
, PPCI_LWZ
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
737 emit_tai(as
, PPCI_LWZ
, uv
, func
,
738 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
742 static void asm_fref(ASMState
*as
, IRIns
*ir
)
744 UNUSED(as
); UNUSED(ir
);
745 lua_assert(!ra_used(ir
));
748 static void asm_strref(ASMState
*as
, IRIns
*ir
)
750 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
751 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
752 int32_t ofs
= (int32_t)sizeof(GCstr
);
754 if (irref_isk(ref
)) {
755 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
756 } else if (!irref_isk(refk
)) {
757 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
758 IRIns
*irr
= IR(ir
->op2
);
759 if (ra_hasreg(irr
->r
)) {
760 ra_noweak(as
, irr
->r
);
762 } else if (mayfuse(as
, irr
->op2
) &&
763 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
764 checki16(ofs
+ IR(irr
->op2
)->i
)) {
765 ofs
+= IR(irr
->op2
)->i
;
766 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
768 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
770 emit_tai(as
, PPCI_ADDI
, dest
, dest
, ofs
);
771 emit_tab(as
, PPCI_ADD
, dest
, left
, right
);
774 r
= ra_alloc1(as
, ref
, RSET_GPR
);
777 emit_tai(as
, PPCI_ADDI
, dest
, r
, ofs
);
779 emit_tab(as
, PPCI_ADD
, dest
, r
,
780 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
783 /* -- Loads and stores ---------------------------------------------------- */
785 static PPCIns
asm_fxloadins(IRIns
*ir
)
787 switch (irt_type(ir
->t
)) {
788 case IRT_I8
: return PPCI_LBZ
; /* Needs sign-extension. */
789 case IRT_U8
: return PPCI_LBZ
;
790 case IRT_I16
: return PPCI_LHA
;
791 case IRT_U16
: return PPCI_LHZ
;
792 case IRT_NUM
: return PPCI_LFD
;
793 case IRT_FLOAT
: return PPCI_LFS
;
794 default: return PPCI_LWZ
;
798 static PPCIns
asm_fxstoreins(IRIns
*ir
)
800 switch (irt_type(ir
->t
)) {
801 case IRT_I8
: case IRT_U8
: return PPCI_STB
;
802 case IRT_I16
: case IRT_U16
: return PPCI_STH
;
803 case IRT_NUM
: return PPCI_STFD
;
804 case IRT_FLOAT
: return PPCI_STFS
;
805 default: return PPCI_STW
;
809 static void asm_fload(ASMState
*as
, IRIns
*ir
)
811 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
812 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
813 PPCIns pi
= asm_fxloadins(ir
);
815 if (ir
->op2
== IRFL_TAB_ARRAY
) {
816 ofs
= asm_fuseabase(as
, ir
->op1
);
817 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
818 emit_tai(as
, PPCI_ADDI
, dest
, idx
, ofs
);
822 ofs
= field_ofs
[ir
->op2
];
823 lua_assert(!irt_isi8(ir
->t
));
824 emit_tai(as
, pi
, dest
, idx
, ofs
);
827 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
829 if (ir
->r
!= RID_SINK
) {
830 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
831 IRIns
*irf
= IR(ir
->op1
);
832 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
833 int32_t ofs
= field_ofs
[irf
->op2
];
834 PPCIns pi
= asm_fxstoreins(ir
);
835 emit_tai(as
, pi
, src
, idx
, ofs
);
839 static void asm_xload(ASMState
*as
, IRIns
*ir
)
841 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
842 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
844 emit_as(as
, PPCI_EXTSB
, dest
, dest
);
845 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
848 static void asm_xstore_(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
851 if (ir
->r
== RID_SINK
)
853 if (ofs
== 0 && mayfuse(as
, ir
->op2
) && (irb
= IR(ir
->op2
))->o
== IR_BSWAP
&&
854 ra_noreg(irb
->r
) && (irt_isint(ir
->t
) || irt_isu32(ir
->t
))) {
855 /* Fuse BSWAP with XSTORE to stwbrx. */
856 Reg src
= ra_alloc1(as
, irb
->op1
, RSET_GPR
);
857 asm_fusexrefx(as
, PPCI_STWBRX
, src
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
859 Reg src
= ra_alloc1(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
860 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
861 rset_exclude(RSET_GPR
, src
), ofs
);
865 #define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
867 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
870 Reg dest
= RID_NONE
, type
= RID_TMP
, tmp
= RID_TMP
, idx
;
871 RegSet allow
= RSET_GPR
;
872 int32_t ofs
= AHUREF_LSX
;
874 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
875 if (!irt_isnum(t
)) ofs
= 0;
876 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
877 rset_clear(allow
, dest
);
879 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
881 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, rset_exclude(allow
, idx
));
882 asm_guardcc(as
, CC_GE
);
883 emit_ab(as
, PPCI_CMPLW
, type
, tisnum
);
884 if (ra_hasreg(dest
)) {
885 if (ofs
== AHUREF_LSX
) {
886 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
,
887 (idx
&255)), (idx
>>8)));
888 emit_fab(as
, PPCI_LFDX
, dest
, (idx
&255), tmp
);
890 emit_fai(as
, PPCI_LFD
, dest
, idx
, ofs
);
894 asm_guardcc(as
, CC_NE
);
895 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(t
));
896 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, idx
, ofs
+4);
898 if (ofs
== AHUREF_LSX
) {
899 emit_tab(as
, PPCI_LWZX
, type
, (idx
&255), tmp
);
900 emit_slwi(as
, tmp
, (idx
>>8), 3);
902 emit_tai(as
, PPCI_LWZ
, type
, idx
, ofs
);
906 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
908 RegSet allow
= RSET_GPR
;
909 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
910 int32_t ofs
= AHUREF_LSX
;
911 if (ir
->r
== RID_SINK
)
913 if (irt_isnum(ir
->t
)) {
914 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
916 if (!irt_ispri(ir
->t
)) {
917 src
= ra_alloc1(as
, ir
->op2
, allow
);
918 rset_clear(allow
, src
);
921 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
922 rset_clear(allow
, type
);
924 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
925 if (irt_isnum(ir
->t
)) {
926 if (ofs
== AHUREF_LSX
) {
927 emit_fab(as
, PPCI_STFDX
, src
, (idx
&255), RID_TMP
);
928 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
930 emit_fai(as
, PPCI_STFD
, src
, idx
, ofs
);
934 emit_tai(as
, PPCI_STW
, src
, idx
, ofs
+4);
935 if (ofs
== AHUREF_LSX
) {
936 emit_tab(as
, PPCI_STWX
, type
, (idx
&255), RID_TMP
);
937 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
939 emit_tai(as
, PPCI_STW
, type
, idx
, ofs
);
944 static void asm_sload(ASMState
*as
, IRIns
*ir
)
946 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 0 : 4);
948 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
949 RegSet allow
= RSET_GPR
;
950 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
951 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
952 lua_assert(LJ_DUALNUM
||
953 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
954 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
955 dest
= ra_scratch(as
, RSET_FPR
);
956 asm_tointg(as
, ir
, dest
);
957 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
958 } else if (ra_used(ir
)) {
959 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
960 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
961 rset_clear(allow
, dest
);
962 base
= ra_alloc1(as
, REF_BASE
, allow
);
963 rset_clear(allow
, base
);
964 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
966 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
967 dest
= ra_scratch(as
, RSET_FPR
);
968 emit_fai(as
, PPCI_STFD
, dest
, RID_SP
, SPOFS_TMP
);
969 emit_fb(as
, PPCI_FCTIWZ
, dest
, dest
);
970 t
.irt
= IRT_NUM
; /* Check for original type. */
972 Reg tmp
= ra_scratch(as
, allow
);
973 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, tmp
));
974 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
975 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
976 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
977 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
978 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
979 rset_clear(allow
, hibias
));
980 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPLO
);
981 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
982 emit_asi(as
, PPCI_XORIS
, tmp
, tmp
, 0x8000);
984 t
.irt
= IRT_INT
; /* Check for original type. */
989 base
= ra_alloc1(as
, REF_BASE
, allow
);
990 rset_clear(allow
, base
);
993 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
994 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
995 asm_guardcc(as
, CC_GE
);
996 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tisnum
);
999 if (ra_hasreg(dest
)) emit_fai(as
, PPCI_LFD
, dest
, base
, ofs
-4);
1001 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1002 asm_guardcc(as
, CC_NE
);
1003 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, irt_toitype(t
));
1006 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, base
, ofs
);
1008 if (ra_hasreg(type
)) emit_tai(as
, PPCI_LWZ
, type
, base
, ofs
-4);
1011 /* -- Allocations --------------------------------------------------------- */
1014 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1016 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1017 CTypeID id
= (CTypeID
)IR(ir
->op1
)->i
;
1019 CTInfo info
= lj_ctype_info(cts
, id
, &sz
);
1020 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1022 RegSet drop
= RSET_SCRATCH
;
1023 lua_assert(sz
!= CTSIZE_INVALID
|| (ir
->o
== IR_CNEW
&& ir
->op2
!= REF_NIL
));
1026 if (ra_hasreg(ir
->r
))
1027 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1028 ra_evictset(as
, drop
);
1030 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1032 /* Initialize immutable cdata object. */
1033 if (ir
->o
== IR_CNEWI
) {
1034 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1035 int32_t ofs
= sizeof(GCcdata
);
1036 lua_assert(sz
== 4 || sz
== 8);
1039 lua_assert((ir
+1)->o
== IR_HIOP
);
1042 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1043 emit_tai(as
, PPCI_STW
, r
, RID_RET
, ofs
);
1044 rset_clear(allow
, r
);
1045 if (ofs
== sizeof(GCcdata
)) break;
1048 } else if (ir
->op2
!= REF_NIL
) { /* Create VLA/VLS/aligned cdata. */
1049 ci
= &lj_ir_callinfo
[IRCALL_lj_cdata_newv
];
1050 args
[0] = ASMREF_L
; /* lua_State *L */
1051 args
[1] = ir
->op1
; /* CTypeID id */
1052 args
[2] = ir
->op2
; /* CTSize sz */
1053 args
[3] = ASMREF_TMP1
; /* CTSize align */
1054 asm_gencall(as
, ci
, args
);
1055 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP1
), (int32_t)ctype_align(info
));
1059 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1060 emit_tai(as
, PPCI_STB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1061 emit_tai(as
, PPCI_STH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1062 emit_ti(as
, PPCI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1063 emit_ti(as
, PPCI_LI
, RID_TMP
, id
); /* Lower 16 bit used. Sign-ext ok. */
1064 args
[0] = ASMREF_L
; /* lua_State *L */
1065 args
[1] = ASMREF_TMP1
; /* MSize size */
1066 asm_gencall(as
, ci
, args
);
1067 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1068 ra_releasetmp(as
, ASMREF_TMP1
));
1071 #define asm_cnew(as, ir) ((void)0)
1074 /* -- Write barriers ------------------------------------------------------ */
1076 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1078 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1079 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1081 MCLabel l_end
= emit_label(as
);
1082 emit_tai(as
, PPCI_STW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1083 emit_tai(as
, PPCI_STB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1084 emit_setgl(as
, tab
, gc
.grayagain
);
1085 lua_assert(LJ_GC_BLACK
== 0x04);
1086 emit_rot(as
, PPCI_RLWINM
, mark
, mark
, 0, 30, 28); /* Clear black bit. */
1087 emit_getgl(as
, link
, gc
.grayagain
);
1088 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1089 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, mark
, LJ_GC_BLACK
);
1090 emit_tai(as
, PPCI_LBZ
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1093 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1095 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1099 /* No need for other object barriers (yet). */
1100 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1101 ra_evictset(as
, RSET_SCRATCH
);
1102 l_end
= emit_label(as
);
1103 args
[0] = ASMREF_TMP1
; /* global_State *g */
1104 args
[1] = ir
->op1
; /* TValue *tv */
1105 asm_gencall(as
, ci
, args
);
1106 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1107 obj
= IR(ir
->op1
)->r
;
1108 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1109 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1110 emit_asi(as
, PPCI_ANDIDOT
, tmp
, tmp
, LJ_GC_BLACK
);
1111 emit_condbranch(as
, PPCI_BC
, CC_EQ
, l_end
);
1112 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1113 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1114 emit_tai(as
, PPCI_LBZ
, tmp
, obj
,
1115 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1116 emit_tai(as
, PPCI_LBZ
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1119 /* -- Arithmetic and logic operations ------------------------------------- */
1121 static void asm_fparith(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1123 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1124 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1125 right
= (left
>> 8); left
&= 255;
1126 if (pi
== PPCI_FMUL
)
1127 emit_fac(as
, pi
, dest
, left
, right
);
1129 emit_fab(as
, pi
, dest
, left
, right
);
1132 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1134 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1135 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1136 emit_fb(as
, pi
, dest
, left
);
1139 static void asm_fpmath(ASMState
*as
, IRIns
*ir
)
1141 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
1143 if (ir
->op2
== IRFPM_SQRT
&& (as
->flags
& JIT_F_SQRT
))
1144 asm_fpunary(as
, ir
, PPCI_FSQRT
);
1146 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
1149 static void asm_add(ASMState
*as
, IRIns
*ir
)
1151 if (irt_isnum(ir
->t
)) {
1152 if (!asm_fusemadd(as
, ir
, PPCI_FMADD
, PPCI_FMADD
))
1153 asm_fparith(as
, ir
, PPCI_FADD
);
1155 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1156 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1158 if (irref_isk(ir
->op2
)) {
1159 int32_t k
= IR(ir
->op2
)->i
;
1162 /* May fail due to spills/restores above, but simplifies the logic. */
1163 if (as
->flagmcp
== as
->mcp
) {
1168 emit_tai(as
, pi
, dest
, left
, k
);
1170 } else if ((k
& 0xffff) == 0) {
1171 emit_tai(as
, PPCI_ADDIS
, dest
, left
, (k
>> 16));
1173 } else if (!as
->sectref
) {
1174 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (k
+ 32768) >> 16);
1175 emit_tai(as
, PPCI_ADDI
, dest
, left
, k
);
1180 /* May fail due to spills/restores above, but simplifies the logic. */
1181 if (as
->flagmcp
== as
->mcp
) {
1186 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1187 emit_tab(as
, pi
, dest
, left
, right
);
1191 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1193 if (irt_isnum(ir
->t
)) {
1194 if (!asm_fusemadd(as
, ir
, PPCI_FMSUB
, PPCI_FNMSUB
))
1195 asm_fparith(as
, ir
, PPCI_FSUB
);
1197 PPCIns pi
= PPCI_SUBF
;
1198 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1200 if (irref_isk(ir
->op1
)) {
1201 int32_t k
= IR(ir
->op1
)->i
;
1203 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1204 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1208 /* May fail due to spills/restores above, but simplifies the logic. */
1209 if (as
->flagmcp
== as
->mcp
) {
1214 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1215 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1216 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1220 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1222 if (irt_isnum(ir
->t
)) {
1223 asm_fparith(as
, ir
, PPCI_FMUL
);
1225 PPCIns pi
= PPCI_MULLW
;
1226 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1227 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1228 if (irref_isk(ir
->op2
)) {
1229 int32_t k
= IR(ir
->op2
)->i
;
1231 emit_tai(as
, PPCI_MULLI
, dest
, left
, k
);
1235 /* May fail due to spills/restores above, but simplifies the logic. */
1236 if (as
->flagmcp
== as
->mcp
) {
1241 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1242 emit_tab(as
, pi
, dest
, left
, right
);
1246 #define asm_div(as, ir) asm_fparith(as, ir, PPCI_FDIV)
1247 #define asm_mod(as, ir) asm_callid(as, ir, IRCALL_lj_vm_modi)
1248 #define asm_pow(as, ir) asm_callid(as, ir, IRCALL_lj_vm_powi)
1250 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1252 if (irt_isnum(ir
->t
)) {
1253 asm_fpunary(as
, ir
, PPCI_FNEG
);
1256 PPCIns pi
= PPCI_NEG
;
1257 if (as
->flagmcp
== as
->mcp
) {
1262 dest
= ra_dest(as
, ir
, RSET_GPR
);
1263 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1264 emit_tab(as
, pi
, dest
, left
, 0);
1268 #define asm_abs(as, ir) asm_fpunary(as, ir, PPCI_FABS)
1269 #define asm_atan2(as, ir) asm_callid(as, ir, IRCALL_atan2)
1270 #define asm_ldexp(as, ir) asm_callid(as, ir, IRCALL_ldexp)
1272 static void asm_arithov(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1274 Reg dest
, left
, right
;
1275 if (as
->flagmcp
== as
->mcp
) {
1279 asm_guardcc(as
, CC_SO
);
1280 dest
= ra_dest(as
, ir
, RSET_GPR
);
1281 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1282 right
= (left
>> 8); left
&= 255;
1283 if (pi
== PPCI_SUBFO
) { Reg tmp
= left
; left
= right
; right
= tmp
; }
1284 emit_tab(as
, pi
|PPCF_DOT
, dest
, left
, right
);
1287 #define asm_addov(as, ir) asm_arithov(as, ir, PPCI_ADDO)
1288 #define asm_subov(as, ir) asm_arithov(as, ir, PPCI_SUBFO)
1289 #define asm_mulov(as, ir) asm_arithov(as, ir, PPCI_MULLWO)
1292 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1294 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1295 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1296 PPCIns pi
= PPCI_ADDE
;
1297 if (irref_isk(ir
->op2
)) {
1298 int32_t k
= IR(ir
->op2
)->i
;
1308 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1310 emit_tab(as
, pi
, dest
, left
, right
);
1312 dest
= ra_dest(as
, ir
, RSET_GPR
);
1313 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1314 if (irref_isk(ir
->op2
)) {
1315 int32_t k
= IR(ir
->op2
)->i
;
1317 emit_tai(as
, PPCI_ADDIC
, dest
, left
, k
);
1321 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1322 emit_tab(as
, PPCI_ADDC
, dest
, left
, right
);
1325 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1327 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1328 Reg left
, right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1329 PPCIns pi
= PPCI_SUBFE
;
1330 if (irref_isk(ir
->op1
)) {
1331 int32_t k
= IR(ir
->op1
)->i
;
1341 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1343 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1345 dest
= ra_dest(as
, ir
, RSET_GPR
);
1346 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1347 if (irref_isk(ir
->op1
)) {
1348 int32_t k
= IR(ir
->op1
)->i
;
1350 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1354 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1355 emit_tab(as
, PPCI_SUBFC
, dest
, right
, left
);
1358 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1360 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1361 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1362 emit_tab(as
, PPCI_SUBFZE
, dest
, left
, 0);
1364 dest
= ra_dest(as
, ir
, RSET_GPR
);
1365 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1366 emit_tai(as
, PPCI_SUBFIC
, dest
, left
, 0);
1370 static void asm_bnot(ASMState
*as
, IRIns
*ir
)
1372 Reg dest
, left
, right
;
1373 PPCIns pi
= PPCI_NOR
;
1374 if (as
->flagmcp
== as
->mcp
) {
1379 dest
= ra_dest(as
, ir
, RSET_GPR
);
1380 if (mayfuse(as
, ir
->op1
)) {
1381 IRIns
*irl
= IR(ir
->op1
);
1382 if (irl
->o
== IR_BAND
)
1383 pi
^= (PPCI_NOR
^ PPCI_NAND
);
1384 else if (irl
->o
== IR_BXOR
)
1385 pi
^= (PPCI_NOR
^ PPCI_EQV
);
1386 else if (irl
->o
!= IR_BOR
)
1388 left
= ra_hintalloc(as
, irl
->op1
, dest
, RSET_GPR
);
1389 right
= ra_alloc1(as
, irl
->op2
, rset_exclude(RSET_GPR
, left
));
1392 left
= right
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1394 emit_asb(as
, pi
, dest
, left
, right
);
1397 static void asm_bswap(ASMState
*as
, IRIns
*ir
)
1399 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1401 if (mayfuse(as
, ir
->op1
) && (irx
= IR(ir
->op1
))->o
== IR_XLOAD
&&
1402 ra_noreg(irx
->r
) && (irt_isint(irx
->t
) || irt_isu32(irx
->t
))) {
1403 /* Fuse BSWAP with XLOAD to lwbrx. */
1404 asm_fusexrefx(as
, PPCI_LWBRX
, dest
, irx
->op1
, RSET_GPR
);
1406 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1410 emit_mr(as
, dest
, RID_TMP
);
1412 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 16, 23);
1413 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 0, 7);
1414 emit_rotlwi(as
, tmp
, left
, 8);
1418 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1419 static void asm_fuseandsh(ASMState
*as
, PPCIns pi
, int32_t mask
, IRRef ref
)
1423 if (mayfuse(as
, ref
) && (ir
= IR(ref
), ra_noreg(ir
->r
)) &&
1424 irref_isk(ir
->op2
) && ir
->o
>= IR_BSHL
&& ir
->o
<= IR_BROR
) {
1425 int32_t sh
= (IR(ir
->op2
)->i
& 31);
1428 if ((mask
& ((1u<<sh
)-1))) goto nofuse
;
1431 if ((mask
& ~((~0u)>>sh
))) goto nofuse
;
1439 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1440 *--as
->mcp
= pi
| PPCF_T(left
) | PPCF_B(sh
);
1444 left
= ra_alloc1(as
, ref
, RSET_GPR
);
1445 *--as
->mcp
= pi
| PPCF_T(left
);
1448 static void asm_band(ASMState
*as
, IRIns
*ir
)
1450 Reg dest
, left
, right
;
1451 IRRef lref
= ir
->op1
;
1454 if (as
->flagmcp
== as
->mcp
) {
1459 dest
= ra_dest(as
, ir
, RSET_GPR
);
1460 if (irref_isk(ir
->op2
)) {
1461 int32_t k
= IR(ir
->op2
)->i
;
1463 /* First check for a contiguous bitmask as used by rlwinm. */
1464 uint32_t s1
= lj_ffs((uint32_t)k
);
1465 uint32_t k1
= ((uint32_t)k
>> s1
);
1466 if ((k1
& (k1
+1)) == 0) {
1467 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1468 PPCF_MB(31-lj_fls((uint32_t)k
)) | PPCF_ME(31-s1
),
1473 uint32_t s2
= lj_ffs(~(uint32_t)k
);
1474 uint32_t k2
= (~(uint32_t)k
>> s2
);
1475 if ((k2
& (k2
+1)) == 0) {
1476 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1477 PPCF_MB(32-s2
) | PPCF_ME(30-lj_fls(~(uint32_t)k
)),
1484 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1485 emit_asi(as
, PPCI_ANDIDOT
, dest
, left
, k
);
1487 } else if ((k
& 0xffff) == 0) {
1488 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1489 emit_asi(as
, PPCI_ANDISDOT
, dest
, left
, (k
>> 16));
1494 if (mayfuse(as
, op2
) && IR(op2
)->o
== IR_BNOT
&& ra_noreg(IR(op2
)->r
)) {
1495 dot
^= (PPCI_AND
^ PPCI_ANDC
);
1498 left
= ra_hintalloc(as
, lref
, dest
, RSET_GPR
);
1499 right
= ra_alloc1(as
, op2
, rset_exclude(RSET_GPR
, left
));
1500 emit_asb(as
, PPCI_AND
^ dot
, dest
, left
, right
);
1503 static void asm_bitop(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1505 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1506 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1507 if (irref_isk(ir
->op2
)) {
1508 int32_t k
= IR(ir
->op2
)->i
;
1510 if ((checku16(k
) || (k
& 0xffff) == 0) || (tmp
= dest
, !as
->sectref
)) {
1512 emit_asi(as
, pik
^ (PPCI_ORI
^ PPCI_ORIS
), dest
, tmp
, (k
>> 16));
1513 if ((k
& 0xffff) == 0) return;
1515 emit_asi(as
, pik
, dest
, left
, k
);
1519 /* May fail due to spills/restores above, but simplifies the logic. */
1520 if (as
->flagmcp
== as
->mcp
) {
1525 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1526 emit_asb(as
, pi
, dest
, left
, right
);
1529 #define asm_bor(as, ir) asm_bitop(as, ir, PPCI_OR, PPCI_ORI)
1530 #define asm_bxor(as, ir) asm_bitop(as, ir, PPCI_XOR, PPCI_XORI)
1532 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1536 if (as
->flagmcp
== as
->mcp
) {
1541 dest
= ra_dest(as
, ir
, RSET_GPR
);
1542 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1543 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1544 int32_t shift
= (IR(ir
->op2
)->i
& 31);
1545 if (pik
== 0) /* SLWI */
1546 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, shift
, 0, 31-shift
);
1547 else if (pik
== 1) /* SRWI */
1548 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, (32-shift
)&31, shift
, 31);
1550 emit_asb(as
, pik
|dot
, dest
, left
, shift
);
1552 Reg right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1553 emit_asb(as
, pi
|dot
, dest
, left
, right
);
1557 #define asm_bshl(as, ir) asm_bitshift(as, ir, PPCI_SLW, 0)
1558 #define asm_bshr(as, ir) asm_bitshift(as, ir, PPCI_SRW, 1)
1559 #define asm_bsar(as, ir) asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI)
1560 #define asm_brol(as, ir) \
1561 asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31), \
1562 PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31))
1563 #define asm_bror(as, ir) lua_assert(0)
1565 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1567 if (irt_isnum(ir
->t
)) {
1568 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1570 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1571 right
= (left
>> 8); left
&= 255;
1572 if (tmp
== left
|| tmp
== right
)
1573 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_FPR
,
1574 dest
), left
), right
));
1575 emit_facb(as
, PPCI_FSEL
, dest
, tmp
,
1576 ismax
? left
: right
, ismax
? right
: left
);
1577 emit_fab(as
, PPCI_FSUB
, tmp
, left
, right
);
1579 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1580 Reg tmp1
= RID_TMP
, tmp2
= dest
;
1581 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1582 right
= (left
>> 8); left
&= 255;
1583 if (tmp2
== left
|| tmp2
== right
)
1584 tmp2
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
,
1585 dest
), left
), right
));
1586 emit_tab(as
, PPCI_ADD
, dest
, tmp2
, right
);
1587 emit_asb(as
, ismax
? PPCI_ANDC
: PPCI_AND
, tmp2
, tmp2
, tmp1
);
1588 emit_tab(as
, PPCI_SUBFE
, tmp1
, tmp1
, tmp1
);
1589 emit_tab(as
, PPCI_SUBFC
, tmp2
, tmp2
, tmp1
);
1590 emit_asi(as
, PPCI_XORIS
, tmp2
, right
, 0x8000);
1591 emit_asi(as
, PPCI_XORIS
, tmp1
, left
, 0x8000);
1595 #define asm_min(as, ir) asm_min_max(as, ir, 0)
1596 #define asm_max(as, ir) asm_min_max(as, ir, 1)
1598 /* -- Comparisons --------------------------------------------------------- */
1600 #define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
1601 #define CC_TWO 0x80 /* Check two flags for FP comparison. */
1603 /* Map of comparisons to flags. ORDER IR. */
1604 static const uint8_t asm_compmap
[IR_ABC
+1] = {
1605 /* op int cc FP cc */
1606 /* LT */ CC_GE
+ (CC_GE
<<4),
1607 /* GE */ CC_LT
+ (CC_LE
<<4) + CC_TWO
,
1608 /* LE */ CC_GT
+ (CC_GE
<<4) + CC_TWO
,
1609 /* GT */ CC_LE
+ (CC_LE
<<4),
1610 /* ULT */ CC_GE
+ CC_UNSIGNED
+ (CC_GT
<<4) + CC_TWO
,
1611 /* UGE */ CC_LT
+ CC_UNSIGNED
+ (CC_LT
<<4),
1612 /* ULE */ CC_GT
+ CC_UNSIGNED
+ (CC_GT
<<4),
1613 /* UGT */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
,
1614 /* EQ */ CC_NE
+ (CC_NE
<<4),
1615 /* NE */ CC_EQ
+ (CC_EQ
<<4),
1616 /* ABC */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
/* Same as UGT. */
1619 static void asm_intcomp_(ASMState
*as
, IRRef lref
, IRRef rref
, Reg cr
, PPCCC cc
)
1621 Reg right
, left
= ra_alloc1(as
, lref
, RSET_GPR
);
1622 if (irref_isk(rref
)) {
1623 int32_t k
= IR(rref
)->i
;
1624 if ((cc
& CC_UNSIGNED
) == 0) { /* Signed comparison with constant. */
1626 emit_tai(as
, PPCI_CMPWI
, cr
, left
, k
);
1627 /* Signed comparison with zero and referencing previous ins? */
1628 if (k
== 0 && lref
== as
->curins
-1)
1629 as
->flagmcp
= as
->mcp
; /* Allow elimination of the compare. */
1631 } else if ((cc
& 3) == (CC_EQ
& 3)) { /* Use CMPLWI for EQ or NE. */
1633 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1635 } else if (!as
->sectref
&& ra_noreg(IR(rref
)->r
)) {
1636 emit_tai(as
, PPCI_CMPLWI
, cr
, RID_TMP
, k
);
1637 emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, (k
>> 16));
1641 } else { /* Unsigned comparison with constant. */
1643 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1648 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, left
));
1649 emit_tab(as
, (cc
& CC_UNSIGNED
) ? PPCI_CMPLW
: PPCI_CMPW
, cr
, left
, right
);
1652 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1654 PPCCC cc
= asm_compmap
[ir
->o
];
1655 if (irt_isnum(ir
->t
)) {
1656 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1657 right
= (left
>> 8); left
&= 255;
1658 asm_guardcc(as
, (cc
>> 4));
1660 emit_tab(as
, PPCI_CROR
, ((cc
>>4)&3), ((cc
>>4)&3), (CC_EQ
&3));
1661 emit_fab(as
, PPCI_FCMPU
, 0, left
, right
);
1663 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1664 if (irref_isk(lref
) && !irref_isk(rref
)) {
1665 /* Swap constants to the right (only for ABC). */
1666 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1667 if ((cc
& 2) == 0) cc
^= 1; /* LT <-> GT, LE <-> GE */
1669 asm_guardcc(as
, cc
);
1670 asm_intcomp_(as
, lref
, rref
, 0, cc
);
1674 #define asm_equal(as, ir) asm_comp(as, ir)
1677 /* 64 bit integer comparisons. */
1678 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1680 PPCCC cc
= asm_compmap
[(ir
-1)->o
];
1681 if ((cc
&3) == (CC_EQ
&3)) {
1682 asm_guardcc(as
, cc
);
1683 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CROR
,
1684 (CC_EQ
&3), (CC_EQ
&3), 4+(CC_EQ
&3));
1686 asm_guardcc(as
, CC_EQ
);
1687 emit_tab(as
, PPCI_CROR
, (CC_EQ
&3), (CC_EQ
&3), ((cc
^~(cc
>>2))&1));
1688 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CRANDC
,
1689 (CC_EQ
&3), (CC_EQ
&3), 4+(cc
&3));
1691 /* Loword comparison sets cr1 and is unsigned, except for equality. */
1692 asm_intcomp_(as
, (ir
-1)->op1
, (ir
-1)->op2
, 4,
1693 cc
| ((cc
&3) == (CC_EQ
&3) ? 0 : CC_UNSIGNED
));
1694 /* Hiword comparison sets cr0. */
1695 asm_intcomp_(as
, ir
->op1
, ir
->op2
, 0, cc
);
1696 as
->flagmcp
= NULL
; /* Doesn't work here. */
1700 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1702 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1703 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1706 /* HIOP is marked as a store because it needs its own DCE logic. */
1707 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1708 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1709 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1710 as
->curins
--; /* Always skip the CONV. */
1714 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1715 as
->curins
--; /* Always skip the loword comparison. */
1718 } else if ((ir
-1)->o
== IR_XSTORE
) {
1719 as
->curins
--; /* Handle both stores here. */
1720 if ((ir
-1)->r
!= RID_SINK
) {
1721 asm_xstore_(as
, ir
, 0);
1722 asm_xstore_(as
, ir
-1, 4);
1726 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1727 switch ((ir
-1)->o
) {
1728 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1729 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1730 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1734 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1737 /* Nothing to do here. Handled by lo op itself. */
1739 default: lua_assert(0); break;
1742 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1746 /* -- Profiling ----------------------------------------------------------- */
1748 static void asm_prof(ASMState
*as
, IRIns
*ir
)
1751 asm_guardcc(as
, CC_NE
);
1752 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, HOOK_PROFILE
);
1753 emit_lsglptr(as
, PPCI_LBZ
, RID_TMP
,
1754 (int32_t)offsetof(global_State
, hookmask
));
1757 /* -- Stack handling ------------------------------------------------------ */
1759 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1760 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1761 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1763 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1764 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1765 rset_clear(allow
, pbase
);
1766 tmp
= allow
? rset_pickbot(allow
) :
1767 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1768 emit_condbranch(as
, PPCI_BC
, CC_LT
, asm_exitstub_addr(as
, exitno
));
1769 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1770 emit_tai(as
, PPCI_LWZ
, tmp
, RID_SP
, SPOFS_TMPW
);
1772 ra_modified(as
, tmp
);
1773 emit_ai(as
, PPCI_CMPLWI
, RID_TMP
, (int32_t)(8*topslot
));
1774 emit_tab(as
, PPCI_SUBF
, RID_TMP
, pbase
, tmp
);
1775 emit_tai(as
, PPCI_LWZ
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1776 if (pbase
== RID_TMP
)
1777 emit_getgl(as
, RID_TMP
, jit_base
);
1778 emit_getgl(as
, tmp
, cur_L
);
1779 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1780 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPW
);
1783 /* Restore Lua stack from on-trace state. */
1784 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1786 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1787 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1788 MSize n
, nent
= snap
->nent
;
1789 /* Store the value of all modified slots to the Lua stack. */
1790 for (n
= 0; n
< nent
; n
++) {
1791 SnapEntry sn
= map
[n
];
1792 BCReg s
= snap_slot(sn
);
1793 int32_t ofs
= 8*((int32_t)s
-1);
1794 IRRef ref
= snap_ref(sn
);
1795 IRIns
*ir
= IR(ref
);
1796 if ((sn
& SNAP_NORESTORE
))
1798 if (irt_isnum(ir
->t
)) {
1799 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1800 emit_fai(as
, PPCI_STFD
, src
, RID_BASE
, ofs
);
1803 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1804 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1805 if (!irt_ispri(ir
->t
)) {
1806 Reg src
= ra_alloc1(as
, ref
, allow
);
1807 rset_clear(allow
, src
);
1808 emit_tai(as
, PPCI_STW
, src
, RID_BASE
, ofs
+4);
1810 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1811 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1812 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1814 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1816 emit_tai(as
, PPCI_STW
, type
, RID_BASE
, ofs
);
1820 lua_assert(map
+ nent
== flinks
);
1823 /* -- GC handling --------------------------------------------------------- */
1825 /* Check GC threshold and do one or more GC steps. */
1826 static void asm_gc_check(ASMState
*as
)
1828 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1832 ra_evictset(as
, RSET_SCRATCH
);
1833 l_end
= emit_label(as
);
1834 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1835 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
1836 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0);
1837 args
[0] = ASMREF_TMP1
; /* global_State *g */
1838 args
[1] = ASMREF_TMP2
; /* MSize steps */
1839 asm_gencall(as
, ci
, args
);
1840 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1841 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1842 emit_loadi(as
, tmp
, as
->gcsteps
);
1843 /* Jump around GC step if GC total < GC threshold. */
1844 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_LT
, l_end
);
1845 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tmp
);
1846 emit_getgl(as
, tmp
, gc
.threshold
);
1847 emit_getgl(as
, RID_TMP
, gc
.total
);
1852 /* -- Loop handling ------------------------------------------------------- */
1854 /* Fixup the loop branch. */
1855 static void asm_loop_fixup(ASMState
*as
)
1857 MCode
*p
= as
->mctop
;
1858 MCode
*target
= as
->mcp
;
1859 if (as
->loopinv
) { /* Inverted loop branch? */
1860 /* asm_guardcc already inverted the cond branch and patched the final b. */
1861 p
[-2] = (p
[-2] & (0xffff0000u
& ~PPCF_Y
)) | (((target
-p
+2) & 0x3fffu
) << 2);
1863 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1867 /* -- Head of trace ------------------------------------------------------- */
1869 /* Coalesce BASE register for a root trace. */
1870 static void asm_head_root_base(ASMState
*as
)
1872 IRIns
*ir
= IR(REF_BASE
);
1876 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1877 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1879 emit_mr(as
, r
, RID_BASE
);
1883 /* Coalesce BASE register for a side trace. */
1884 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1886 IRIns
*ir
= IR(REF_BASE
);
1890 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1891 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1893 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1894 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1895 rset_clear(allow
, irp
->r
);
1896 emit_mr(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1898 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1904 /* -- Tail of trace ------------------------------------------------------- */
1906 /* Fixup the tail code. */
1907 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1909 MCode
*p
= as
->mctop
;
1911 int32_t spadj
= as
->T
->spadjust
;
1917 /* Patch stack adjustment. */
1918 lua_assert(checki16(CFRAME_SIZE
+spadj
));
1919 p
[-3] = PPCI_ADDI
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | (CFRAME_SIZE
+spadj
);
1920 p
[-2] = PPCI_STWU
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | spadj
;
1922 /* Patch exit branch. */
1923 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1924 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1927 /* Prepare tail of code. */
1928 static void asm_tail_prep(ASMState
*as
)
1930 MCode
*p
= as
->mctop
- 1; /* Leave room for exit branch. */
1932 as
->invmcp
= as
->mcp
= p
;
1934 as
->mcp
= p
-2; /* Leave room for stack pointer adjustment. */
1939 /* -- Trace setup --------------------------------------------------------- */
1941 /* Ensure there are enough stack slots for call arguments. */
1942 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
1944 IRRef args
[CCI_NARGS_MAX
*2];
1945 uint32_t i
, nargs
= CCI_XNARGS(ci
);
1946 int nslots
= 2, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
1947 asm_collectargs(as
, ir
, ci
, args
);
1948 for (i
= 0; i
< nargs
; i
++)
1949 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
1950 if (nfpr
> 0) nfpr
--; else nslots
= (nslots
+3) & ~1;
1952 if (ngpr
> 0) ngpr
--; else nslots
++;
1954 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
1955 as
->evenspill
= nslots
;
1956 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
1959 static void asm_setup_target(ASMState
*as
)
1961 asm_exitstub_setup(as
, as
->T
->nsnap
+ (as
->parent
? 1 : 0));
1964 /* -- Trace patching ------------------------------------------------------ */
1966 /* Patch exit jumps of existing machine code to a new target. */
1967 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
1969 MCode
*p
= T
->mcode
;
1970 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
1971 MCode
*px
= exitstub_trace_addr(T
, exitno
);
1972 MCode
*cstart
= NULL
;
1973 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
1975 for (; p
< pe
; p
++) {
1976 /* Look for exitstub branch, try to replace with branch to target. */
1978 if ((ins
& 0xfc000000u
) == 0x40000000u
&&
1979 ((ins
^ ((char *)px
-(char *)p
)) & 0xffffu
) == 0) {
1980 ptrdiff_t delta
= (char *)target
- (char *)p
;
1981 if (((ins
>> 16) & 3) == (CC_SO
&3)) {
1982 clearso
= sizeof(MCode
);
1983 delta
-= sizeof(MCode
);
1985 /* Many, but not all short-range branches can be patched directly. */
1986 if (((delta
+ 0x8000) >> 16) == 0) {
1987 *p
= (ins
& 0xffdf0000u
) | ((uint32_t)delta
& 0xffffu
) |
1988 ((delta
& 0x8000) * (PPCF_Y
/0x8000));
1989 if (!cstart
) cstart
= p
;
1991 } else if ((ins
& 0xfc000000u
) == PPCI_B
&&
1992 ((ins
^ ((char *)px
-(char *)p
)) & 0x03ffffffu
) == 0) {
1993 ptrdiff_t delta
= (char *)target
- (char *)p
;
1994 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
1995 *p
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
1996 if (!cstart
) cstart
= p
;
1999 { /* Always patch long-range branch in exit stub itself. */
2000 ptrdiff_t delta
= (char *)target
- (char *)px
- clearso
;
2001 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2002 *px
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2004 if (!cstart
) cstart
= px
;
2005 lj_mcode_sync(cstart
, px
+1);
2006 if (clearso
) { /* Extend the current trace. Ugly workaround. */
2007 MCode
*pp
= J
->cur
.mcode
;
2008 J
->cur
.szmcode
+= sizeof(MCode
);
2009 *--pp
= PPCI_MCRXR
; /* Clear SO flag. */
2011 lj_mcode_sync(pp
, pp
+1);
2013 lj_mcode_patch(J
, mcarea
, 1);