2 ** PPC IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Register allocator extensions --------------------------------------- */
8 /* Allocate a register with a hint. */
9 static Reg
ra_hintalloc(ASMState
*as
, IRRef ref
, Reg hint
, RegSet allow
)
13 if (!ra_hashint(r
) && !iscrossref(as
, ref
))
14 ra_sethint(IR(ref
)->r
, hint
); /* Propagate register hint. */
15 r
= ra_allocref(as
, ref
, allow
);
21 /* Allocate two source registers for three-operand instructions. */
22 static Reg
ra_alloc2(ASMState
*as
, IRIns
*ir
, RegSet allow
)
24 IRIns
*irl
= IR(ir
->op1
), *irr
= IR(ir
->op2
);
25 Reg left
= irl
->r
, right
= irr
->r
;
26 if (ra_hasreg(left
)) {
29 right
= ra_allocref(as
, ir
->op2
, rset_exclude(allow
, left
));
32 } else if (ra_hasreg(right
)) {
34 left
= ra_allocref(as
, ir
->op1
, rset_exclude(allow
, right
));
35 } else if (ra_hashint(right
)) {
36 right
= ra_allocref(as
, ir
->op2
, allow
);
37 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(allow
, right
));
39 left
= ra_allocref(as
, ir
->op1
, allow
);
40 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(allow
, left
));
42 return left
| (right
<< 8);
45 /* -- Guard handling ------------------------------------------------------ */
47 /* Setup exit stubs after the end of each trace. */
48 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
51 MCode
*mxp
= as
->mctop
;
52 if (mxp
- (nexits
+ 3 + MCLIM_REDZONE
) < as
->mclim
)
54 /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
55 for (i
= nexits
-1; (int32_t)i
>= 0; i
--)
56 *--mxp
= PPCI_BL
|(((-3-i
)&0x00ffffffu
)<<2);
57 *--mxp
= PPCI_LI
|PPCF_T(RID_TMP
)|as
->T
->traceno
; /* Read by exit handler. */
59 *mxp
= PPCI_BL
|((((MCode
*)(void *)lj_vm_exit_handler
-mxp
)&0x00ffffffu
)<<2);
60 *--mxp
= PPCI_MFLR
|PPCF_T(RID_TMP
);
64 static MCode
*asm_exitstub_addr(ASMState
*as
, ExitNo exitno
)
66 /* Keep this in-sync with exitstub_trace_addr(). */
67 return as
->mctop
+ exitno
+ 3;
70 /* Emit conditional branch to exit for guard. */
71 static void asm_guardcc(ASMState
*as
, PPCCC cc
)
73 MCode
*target
= asm_exitstub_addr(as
, as
->snapno
);
75 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
77 *p
= PPCI_B
| (((target
-p
) & 0x00ffffffu
) << 2);
78 emit_condbranch(as
, PPCI_BC
, cc
^4, p
);
81 emit_condbranch(as
, PPCI_BC
, cc
, target
);
84 /* -- Operand fusion ------------------------------------------------------ */
86 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
87 #define CONFLICT_SEARCH_LIM 31
89 /* Check if there's no conflicting instruction between curins and ref. */
90 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
)
94 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
95 return 0; /* Give up, ref is too far away. */
97 if (ir
[i
].o
== conflict
)
98 return 0; /* Conflict found. */
99 return 1; /* Ok, no conflict. */
102 /* Fuse the array base of colocated arrays. */
103 static int32_t asm_fuseabase(ASMState
*as
, IRRef ref
)
106 if (ir
->o
== IR_TNEW
&& ir
->op1
<= LJ_MAX_COLOSIZE
&&
107 !neverfuse(as
) && noconflict(as
, ref
, IR_NEWREF
))
108 return (int32_t)sizeof(GCtab
);
112 /* Indicates load/store indexed is ok. */
113 #define AHUREF_LSX ((int32_t)0x80000000)
115 /* Fuse array/hash/upvalue reference into register+offset operand. */
116 static Reg
asm_fuseahuref(ASMState
*as
, IRRef ref
, int32_t *ofsp
, RegSet allow
)
119 if (ra_noreg(ir
->r
)) {
120 if (ir
->o
== IR_AREF
) {
121 if (mayfuse(as
, ref
)) {
122 if (irref_isk(ir
->op2
)) {
123 IRRef tab
= IR(ir
->op1
)->op1
;
124 int32_t ofs
= asm_fuseabase(as
, tab
);
125 IRRef refa
= ofs
? tab
: ir
->op1
;
126 ofs
+= 8*IR(ir
->op2
)->i
;
129 return ra_alloc1(as
, refa
, allow
);
132 if (*ofsp
== AHUREF_LSX
) {
133 Reg base
= ra_alloc1(as
, ir
->op1
, allow
);
134 Reg idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
135 return base
| (idx
<< 8);
138 } else if (ir
->o
== IR_HREFK
) {
139 if (mayfuse(as
, ref
)) {
140 int32_t ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
143 return ra_alloc1(as
, ir
->op1
, allow
);
146 } else if (ir
->o
== IR_UREFC
) {
147 if (irref_isk(ir
->op1
)) {
148 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
149 int32_t ofs
= i32ptr(&gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.tv
);
150 int32_t jgl
= (intptr_t)J2G(as
->J
);
151 if ((uint32_t)(ofs
-jgl
) < 65536) {
152 *ofsp
= ofs
-jgl
-32768;
155 *ofsp
= (int16_t)ofs
;
156 return ra_allock(as
, ofs
-(int16_t)ofs
, allow
);
162 return ra_alloc1(as
, ref
, allow
);
165 /* Fuse XLOAD/XSTORE reference into load/store operand. */
166 static void asm_fusexref(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
167 RegSet allow
, int32_t ofs
)
171 if (ra_noreg(ir
->r
) && canfuse(as
, ir
)) {
172 if (ir
->o
== IR_ADD
) {
174 if (irref_isk(ir
->op2
) && (ofs2
= ofs
+ IR(ir
->op2
)->i
, checki16(ofs2
))) {
177 } else if (ofs
== 0) {
178 Reg right
, left
= ra_alloc2(as
, ir
, allow
);
179 right
= (left
>> 8); left
&= 255;
180 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
183 } else if (ir
->o
== IR_STRREF
) {
184 lua_assert(ofs
== 0);
185 ofs
= (int32_t)sizeof(GCstr
);
186 if (irref_isk(ir
->op2
)) {
187 ofs
+= IR(ir
->op2
)->i
;
189 } else if (irref_isk(ir
->op1
)) {
190 ofs
+= IR(ir
->op1
)->i
;
193 /* NYI: Fuse ADD with constant. */
194 Reg tmp
, right
, left
= ra_alloc2(as
, ir
, allow
);
195 right
= (left
>> 8); left
&= 255;
196 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(allow
, left
), right
));
197 emit_fai(as
, pi
, rt
, tmp
, ofs
);
198 emit_tab(as
, PPCI_ADD
, tmp
, left
, right
);
201 if (!checki16(ofs
)) {
202 Reg left
= ra_alloc1(as
, ref
, allow
);
203 Reg right
= ra_allock(as
, ofs
, rset_exclude(allow
, left
));
204 emit_fab(as
, PPCI_LWZX
| ((pi
>> 20) & 0x780), rt
, left
, right
);
209 base
= ra_alloc1(as
, ref
, allow
);
210 emit_fai(as
, pi
, rt
, base
, ofs
);
213 /* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
214 static void asm_fusexrefx(ASMState
*as
, PPCIns pi
, Reg rt
, IRRef ref
,
217 IRIns
*ira
= IR(ref
);
219 if (canfuse(as
, ira
) && ira
->o
== IR_ADD
&& ra_noreg(ira
->r
)) {
220 left
= ra_alloc2(as
, ira
, allow
);
221 right
= (left
>> 8); left
&= 255;
223 right
= ra_alloc1(as
, ref
, allow
);
226 emit_tab(as
, pi
, rt
, left
, right
);
229 /* Fuse to multiply-add/sub instruction. */
230 static int asm_fusemadd(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pir
)
232 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
235 ((mayfuse(as
, lref
) && (irm
= IR(lref
), irm
->o
== IR_MUL
) &&
237 (mayfuse(as
, rref
) && (irm
= IR(rref
), irm
->o
== IR_MUL
) &&
238 (rref
= lref
, pi
= pir
, ra_noreg(irm
->r
))))) {
239 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
240 Reg add
= ra_alloc1(as
, rref
, RSET_FPR
);
241 Reg right
, left
= ra_alloc2(as
, irm
, rset_exclude(RSET_FPR
, add
));
242 right
= (left
>> 8); left
&= 255;
243 emit_facb(as
, pi
, dest
, left
, right
, add
);
249 /* -- Calls --------------------------------------------------------------- */
251 /* Generate a call to a C function. */
252 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
254 uint32_t n
, nargs
= CCI_NARGS(ci
);
256 Reg gpr
= REGARG_FIRSTGPR
, fpr
= REGARG_FIRSTFPR
;
257 if ((void *)ci
->func
)
258 emit_call(as
, (void *)ci
->func
);
259 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
263 if (irt_isfp(ir
->t
)) {
264 if (fpr
<= REGARG_LASTFPR
) {
265 lua_assert(rset_test(as
->freeset
, fpr
)); /* Already evicted. */
266 ra_leftov(as
, fpr
, ref
);
269 Reg r
= ra_alloc1(as
, ref
, RSET_FPR
);
270 if (irt_isnum(ir
->t
)) ofs
= (ofs
+ 4) & ~4;
271 emit_spstore(as
, ir
, r
, ofs
);
272 ofs
+= irt_isnum(ir
->t
) ? 8 : 4;
275 if (gpr
<= REGARG_LASTGPR
) {
276 lua_assert(rset_test(as
->freeset
, gpr
)); /* Already evicted. */
277 ra_leftov(as
, gpr
, ref
);
280 Reg r
= ra_alloc1(as
, ref
, RSET_GPR
);
281 emit_spstore(as
, ir
, r
, ofs
);
286 if (gpr
<= REGARG_LASTGPR
)
293 if ((ci
->flags
& CCI_VARARG
)) /* Vararg calls need to know about FPR use. */
294 emit_tab(as
, fpr
== REGARG_FIRSTFPR
? PPCI_CRXOR
: PPCI_CREQV
, 6, 6, 6);
297 /* Setup result reg/sp for call. Evict scratch regs. */
298 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
300 RegSet drop
= RSET_SCRATCH
;
301 int hiop
= ((ir
+1)->o
== IR_HIOP
&& !irt_isnil((ir
+1)->t
));
302 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
304 if (ra_hasreg(ir
->r
))
305 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
306 if (hiop
&& ra_hasreg((ir
+1)->r
))
307 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
308 ra_evictset(as
, drop
); /* Evictions must be performed first. */
310 lua_assert(!irt_ispri(ir
->t
));
311 if (irt_isfp(ir
->t
)) {
312 if ((ci
->flags
& CCI_CASTU64
)) {
313 /* Use spill slot or temp slots. */
314 int32_t ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
316 if (ra_hasreg(dest
)) {
318 ra_modified(as
, dest
);
319 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, ofs
);
321 emit_tai(as
, PPCI_STW
, RID_RETHI
, RID_SP
, ofs
);
322 emit_tai(as
, PPCI_STW
, RID_RETLO
, RID_SP
, ofs
+4);
324 ra_destreg(as
, ir
, RID_FPRET
);
329 ra_destreg(as
, ir
, RID_RET
);
334 static void asm_call(ASMState
*as
, IRIns
*ir
)
336 IRRef args
[CCI_NARGS_MAX
];
337 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
338 asm_collectargs(as
, ir
, ci
, args
);
339 asm_setupresult(as
, ir
, ci
);
340 asm_gencall(as
, ci
, args
);
343 static void asm_callx(ASMState
*as
, IRIns
*ir
)
345 IRRef args
[CCI_NARGS_MAX
*2];
349 ci
.flags
= asm_callx_flags(as
, ir
);
350 asm_collectargs(as
, ir
, &ci
, args
);
351 asm_setupresult(as
, ir
, &ci
);
352 func
= ir
->op2
; irf
= IR(func
);
353 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
354 if (irref_isk(func
)) { /* Call to constant address. */
355 ci
.func
= (ASMFunction
)(void *)(irf
->i
);
356 } else { /* Need a non-argument register for indirect calls. */
357 RegSet allow
= RSET_GPR
& ~RSET_RANGE(RID_R0
, REGARG_LASTGPR
+1);
358 Reg freg
= ra_alloc1(as
, func
, allow
);
359 *--as
->mcp
= PPCI_BCTRL
;
360 *--as
->mcp
= PPCI_MTCTR
| PPCF_T(freg
);
361 ci
.func
= (ASMFunction
)(void *)0;
363 asm_gencall(as
, &ci
, args
);
366 static void asm_callid(ASMState
*as
, IRIns
*ir
, IRCallID id
)
368 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
372 asm_setupresult(as
, ir
, ci
);
373 asm_gencall(as
, ci
, args
);
376 /* -- Returns ------------------------------------------------------------- */
378 /* Return to lower frame. Guard that it goes to the right spot. */
379 static void asm_retf(ASMState
*as
, IRIns
*ir
)
381 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
382 void *pc
= ir_kptr(IR(ir
->op2
));
383 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
384 as
->topslot
-= (BCReg
)delta
;
385 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
386 irt_setmark(IR(REF_BASE
)->t
); /* Children must not coalesce with BASE reg. */
387 emit_setgl(as
, base
, jit_base
);
388 emit_addptr(as
, base
, -8*delta
);
389 asm_guardcc(as
, CC_NE
);
390 emit_ab(as
, PPCI_CMPW
, RID_TMP
,
391 ra_allock(as
, i32ptr(pc
), rset_exclude(RSET_GPR
, base
)));
392 emit_tai(as
, PPCI_LWZ
, RID_TMP
, base
, -8);
395 /* -- Type conversions ---------------------------------------------------- */
397 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
399 RegSet allow
= RSET_FPR
;
400 Reg tmp
= ra_scratch(as
, rset_clear(allow
, left
));
401 Reg fbias
= ra_scratch(as
, rset_clear(allow
, tmp
));
402 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
403 Reg hibias
= ra_allock(as
, 0x43300000, rset_exclude(RSET_GPR
, dest
));
404 asm_guardcc(as
, CC_NE
);
405 emit_fab(as
, PPCI_FCMPU
, 0, tmp
, left
);
406 emit_fab(as
, PPCI_FSUB
, tmp
, tmp
, fbias
);
407 emit_fai(as
, PPCI_LFD
, tmp
, RID_SP
, SPOFS_TMP
);
408 emit_tai(as
, PPCI_STW
, RID_TMP
, RID_SP
, SPOFS_TMPLO
);
409 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
410 emit_asi(as
, PPCI_XORIS
, RID_TMP
, dest
, 0x8000);
411 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
412 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
413 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
415 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
416 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
419 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
421 RegSet allow
= RSET_FPR
;
422 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
423 Reg left
= ra_alloc1(as
, ir
->op1
, allow
);
424 Reg right
= ra_alloc1(as
, ir
->op2
, rset_clear(allow
, left
));
425 Reg tmp
= ra_scratch(as
, rset_clear(allow
, right
));
426 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
427 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
428 emit_fab(as
, PPCI_FADD
, tmp
, left
, right
);
431 static void asm_conv(ASMState
*as
, IRIns
*ir
)
433 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
434 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
435 IRRef lref
= ir
->op1
;
436 lua_assert(irt_type(ir
->t
) != st
);
437 lua_assert(!(irt_isint64(ir
->t
) ||
438 (st
== IRT_I64
|| st
== IRT_U64
))); /* Handled by SPLIT. */
439 if (irt_isfp(ir
->t
)) {
440 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
441 if (stfp
) { /* FP to FP conversion. */
442 if (st
== IRT_NUM
) /* double -> float conversion. */
443 emit_fb(as
, PPCI_FRSP
, dest
, ra_alloc1(as
, lref
, RSET_FPR
));
444 else /* float -> double conversion is a no-op on PPC. */
445 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
446 } else { /* Integer to FP conversion. */
447 /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
448 /* IRT_U32: Bias with 2^52, subtract 2^52. */
449 RegSet allow
= RSET_GPR
;
450 Reg left
= ra_alloc1(as
, lref
, allow
);
451 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, left
));
452 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
454 if (irt_isfloat(ir
->t
)) emit_fb(as
, PPCI_FRSP
, dest
, dest
);
455 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
456 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
457 kbias
= (const float *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000));
458 if (st
== IRT_U32
) kbias
++;
459 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31), (void *)kbias
,
460 rset_clear(allow
, hibias
));
461 emit_tai(as
, PPCI_STW
, st
== IRT_U32
? left
: RID_TMP
,
462 RID_SP
, SPOFS_TMPLO
);
463 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
464 if (st
!= IRT_U32
) emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, 0x8000);
466 } else if (stfp
) { /* FP to integer conversion. */
467 if (irt_isguard(ir
->t
)) {
468 /* Checked conversions are only supported from number to int. */
469 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
470 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
472 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
473 Reg left
= ra_alloc1(as
, lref
, RSET_FPR
);
474 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
475 if (irt_isu32(ir
->t
)) {
476 /* Convert both x and x-2^31 to int and merge results. */
477 Reg tmpi
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
478 emit_asb(as
, PPCI_OR
, dest
, dest
, tmpi
); /* Select with mask idiom. */
479 emit_asb(as
, PPCI_AND
, tmpi
, tmpi
, RID_TMP
);
480 emit_asb(as
, PPCI_ANDC
, dest
, dest
, RID_TMP
);
481 emit_tai(as
, PPCI_LWZ
, tmpi
, RID_SP
, SPOFS_TMPLO
); /* tmp = (int)(x) */
482 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, 0x8000); /* dest += 2^31 */
483 emit_asb(as
, PPCI_SRAWI
, RID_TMP
, dest
, 31); /* mask = -(dest < 0) */
484 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
485 emit_tai(as
, PPCI_LWZ
, dest
,
486 RID_SP
, SPOFS_TMPLO
); /* dest = (int)(x-2^31) */
487 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
488 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
489 emit_fb(as
, PPCI_FCTIWZ
, tmp
, tmp
);
490 emit_fab(as
, PPCI_FSUB
, tmp
, left
, tmp
);
491 emit_lsptr(as
, PPCI_LFS
, (tmp
& 31),
492 (void *)lj_ir_k64_find(as
->J
, U64x(4f000000
,00000000)),
495 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
496 emit_fai(as
, PPCI_STFD
, tmp
, RID_SP
, SPOFS_TMP
);
497 emit_fb(as
, PPCI_FCTIWZ
, tmp
, left
);
501 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
502 if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
503 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
504 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
505 if ((ir
->op2
& IRCONV_SEXT
))
506 emit_as(as
, st
== IRT_I8
? PPCI_EXTSB
: PPCI_EXTSH
, dest
, left
);
508 emit_rot(as
, PPCI_RLWINM
, dest
, left
, 0, st
== IRT_U8
? 24 : 16, 31);
509 } else { /* 32/64 bit integer conversions. */
510 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
511 ra_leftov(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
517 static void asm_conv64(ASMState
*as
, IRIns
*ir
)
519 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
520 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
525 args
[1] = (ir
-1)->op1
;
526 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) {
527 id
= IRCALL_fp64_d2l
+ ((st
== IRT_FLOAT
) ? 2 : 0) + (dt
- IRT_I64
);
530 id
= IRCALL_fp64_l2d
+ ((dt
== IRT_FLOAT
) ? 2 : 0) + (st
- IRT_I64
);
532 ci
= &lj_ir_callinfo
[id
];
533 asm_setupresult(as
, ir
, ci
);
534 asm_gencall(as
, ci
, args
);
538 static void asm_strto(ASMState
*as
, IRIns
*ir
)
540 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
543 RegSet drop
= RSET_SCRATCH
;
544 if (ra_hasreg(ir
->r
)) rset_set(drop
, ir
->r
); /* Spill dest reg (if any). */
545 ra_evictset(as
, drop
);
546 asm_guardcc(as
, CC_EQ
);
547 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0); /* Test return status. */
548 args
[0] = ir
->op1
; /* GCstr *str */
549 args
[1] = ASMREF_TMP1
; /* TValue *n */
550 asm_gencall(as
, ci
, args
);
551 /* Store the result to the spill slot or temp slots. */
552 ofs
= ir
->s
? sps_scale(ir
->s
) : SPOFS_TMP
;
553 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_SP
, ofs
);
556 /* Get pointer to TValue. */
557 static void asm_tvptr(ASMState
*as
, Reg dest
, IRRef ref
)
560 if (irt_isnum(ir
->t
)) {
561 if (irref_isk(ref
)) /* Use the number constant itself as a TValue. */
562 ra_allockreg(as
, i32ptr(ir_knum(ir
)), dest
);
563 else /* Otherwise force a spill and use the spill slot. */
564 emit_tai(as
, PPCI_ADDI
, dest
, RID_SP
, ra_spill(as
, ir
));
566 /* Otherwise use g->tmptv to hold the TValue. */
567 RegSet allow
= rset_exclude(RSET_GPR
, dest
);
569 emit_tai(as
, PPCI_ADDI
, dest
, RID_JGL
, offsetof(global_State
, tmptv
)-32768);
570 if (!irt_ispri(ir
->t
)) {
571 Reg src
= ra_alloc1(as
, ref
, allow
);
572 emit_setgl(as
, src
, tmptv
.gcr
);
574 type
= ra_allock(as
, irt_toitype(ir
->t
), allow
);
575 emit_setgl(as
, type
, tmptv
.it
);
579 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
584 if (irt_isnum(IR(ir
->op1
)->t
) || (ir
+1)->o
== IR_HIOP
) {
585 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
586 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
587 asm_setupresult(as
, ir
, ci
); /* GCstr * */
588 asm_gencall(as
, ci
, args
);
589 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op1
);
591 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
592 args
[1] = ir
->op1
; /* int32_t k */
593 asm_setupresult(as
, ir
, ci
); /* GCstr * */
594 asm_gencall(as
, ci
, args
);
598 /* -- Memory references --------------------------------------------------- */
600 static void asm_aref(ASMState
*as
, IRIns
*ir
)
602 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
604 if (irref_isk(ir
->op2
)) {
605 IRRef tab
= IR(ir
->op1
)->op1
;
606 int32_t ofs
= asm_fuseabase(as
, tab
);
607 IRRef refa
= ofs
? tab
: ir
->op1
;
608 ofs
+= 8*IR(ir
->op2
)->i
;
610 base
= ra_alloc1(as
, refa
, RSET_GPR
);
611 emit_tai(as
, PPCI_ADDI
, dest
, base
, ofs
);
615 base
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
616 idx
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, base
));
617 emit_tab(as
, PPCI_ADD
, dest
, RID_TMP
, base
);
618 emit_slwi(as
, RID_TMP
, idx
, 3);
621 /* Inlined hash lookup. Specialized for key type and for const keys.
622 ** The equivalent C code is:
623 ** Node *n = hashkey(t, key);
625 ** if (lj_obj_equal(&n->key, key)) return &n->val;
626 ** } while ((n = nextnode(n)));
629 static void asm_href(ASMState
*as
, IRIns
*ir
, IROp merge
)
631 RegSet allow
= RSET_GPR
;
632 int destused
= ra_used(ir
);
633 Reg dest
= ra_dest(as
, ir
, allow
);
634 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
635 Reg key
= RID_NONE
, tmp1
= RID_TMP
, tmp2
;
636 Reg tisnum
= RID_NONE
, tmpnum
= RID_NONE
;
637 IRRef refkey
= ir
->op2
;
638 IRIns
*irkey
= IR(refkey
);
639 IRType1 kt
= irkey
->t
;
641 MCLabel l_end
, l_loop
, l_next
;
643 rset_clear(allow
, tab
);
645 key
= ra_alloc1(as
, refkey
, RSET_FPR
);
646 tmpnum
= ra_scratch(as
, rset_exclude(RSET_FPR
, key
));
647 tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
648 rset_clear(allow
, tisnum
);
649 } else if (!irt_ispri(kt
)) {
650 key
= ra_alloc1(as
, refkey
, allow
);
651 rset_clear(allow
, key
);
653 tmp2
= ra_scratch(as
, allow
);
654 rset_clear(allow
, tmp2
);
656 /* Key not found in chain: jump to exit (if merged) or load niltv. */
657 l_end
= emit_label(as
);
660 asm_guardcc(as
, CC_EQ
);
662 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
664 /* Follow hash chain until the end. */
666 emit_ai(as
, PPCI_CMPWI
, dest
, 0);
667 emit_tai(as
, PPCI_LWZ
, dest
, dest
, (int32_t)offsetof(Node
, next
));
668 l_next
= emit_label(as
);
670 /* Type and value comparison. */
672 asm_guardcc(as
, CC_EQ
);
674 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
676 emit_fab(as
, PPCI_FCMPU
, 0, tmpnum
, key
);
677 emit_condbranch(as
, PPCI_BC
, CC_GE
, l_next
);
678 emit_ab(as
, PPCI_CMPLW
, tmp1
, tisnum
);
679 emit_fai(as
, PPCI_LFD
, tmpnum
, dest
, (int32_t)offsetof(Node
, key
.n
));
681 if (!irt_ispri(kt
)) {
682 emit_ab(as
, PPCI_CMPW
, tmp2
, key
);
683 emit_condbranch(as
, PPCI_BC
, CC_NE
, l_next
);
685 emit_ai(as
, PPCI_CMPWI
, tmp1
, irt_toitype(irkey
->t
));
687 emit_tai(as
, PPCI_LWZ
, tmp2
, dest
, (int32_t)offsetof(Node
, key
.gcr
));
689 emit_tai(as
, PPCI_LWZ
, tmp1
, dest
, (int32_t)offsetof(Node
, key
.it
));
690 *l_loop
= PPCI_BC
| PPCF_Y
| PPCF_CC(CC_NE
) |
691 (((char *)as
->mcp
-(char *)l_loop
) & 0xffffu
);
693 /* Load main position relative to tab->node into dest. */
694 khash
= irref_isk(refkey
) ? ir_khash(irkey
) : 1;
696 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
699 if (irref_isk(refkey
))
700 tmphash
= ra_allock(as
, khash
, allow
);
701 emit_tab(as
, PPCI_ADD
, dest
, dest
, tmp1
);
702 emit_tai(as
, PPCI_MULLI
, tmp1
, tmp1
, sizeof(Node
));
703 emit_asb(as
, PPCI_AND
, tmp1
, tmp2
, tmphash
);
704 emit_tai(as
, PPCI_LWZ
, dest
, tab
, (int32_t)offsetof(GCtab
, node
));
705 emit_tai(as
, PPCI_LWZ
, tmp2
, tab
, (int32_t)offsetof(GCtab
, hmask
));
706 if (irref_isk(refkey
)) {
708 } else if (irt_isstr(kt
)) {
709 emit_tai(as
, PPCI_LWZ
, tmp1
, key
, (int32_t)offsetof(GCstr
, hash
));
710 } else { /* Must match with hash*() in lj_tab.c. */
711 emit_tab(as
, PPCI_SUBF
, tmp1
, tmp2
, tmp1
);
712 emit_rotlwi(as
, tmp2
, tmp2
, HASH_ROT3
);
713 emit_asb(as
, PPCI_XOR
, tmp1
, tmp1
, tmp2
);
714 emit_rotlwi(as
, tmp1
, tmp1
, (HASH_ROT2
+HASH_ROT1
)&31);
715 emit_tab(as
, PPCI_SUBF
, tmp2
, dest
, tmp2
);
717 int32_t ofs
= ra_spill(as
, irkey
);
718 emit_asb(as
, PPCI_XOR
, tmp2
, tmp2
, tmp1
);
719 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
720 emit_tab(as
, PPCI_ADD
, tmp1
, tmp1
, tmp1
);
721 emit_tai(as
, PPCI_LWZ
, tmp2
, RID_SP
, ofs
+4);
722 emit_tai(as
, PPCI_LWZ
, tmp1
, RID_SP
, ofs
);
724 emit_asb(as
, PPCI_XOR
, tmp2
, key
, tmp1
);
725 emit_rotlwi(as
, dest
, tmp1
, HASH_ROT1
);
726 emit_tai(as
, PPCI_ADDI
, tmp1
, tmp2
, HASH_BIAS
);
727 emit_tai(as
, PPCI_ADDIS
, tmp2
, key
, (HASH_BIAS
+ 32768)>>16);
733 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
735 IRIns
*kslot
= IR(ir
->op2
);
736 IRIns
*irkey
= IR(kslot
->op1
);
737 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
738 int32_t kofs
= ofs
+ (int32_t)offsetof(Node
, key
);
739 Reg dest
= (ra_used(ir
)||ofs
> 32736) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
740 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
741 Reg key
= RID_NONE
, type
= RID_TMP
, idx
= node
;
742 RegSet allow
= rset_exclude(RSET_GPR
, node
);
743 lua_assert(ofs
% sizeof(Node
) == 0);
746 rset_clear(allow
, dest
);
747 kofs
= (int32_t)offsetof(Node
, key
);
748 } else if (ra_hasreg(dest
)) {
749 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
751 asm_guardcc(as
, CC_NE
);
752 if (!irt_ispri(irkey
->t
)) {
753 key
= ra_scratch(as
, allow
);
754 rset_clear(allow
, key
);
756 rset_clear(allow
, type
);
757 if (irt_isnum(irkey
->t
)) {
758 emit_cmpi(as
, key
, (int32_t)ir_knum(irkey
)->u32
.lo
);
759 asm_guardcc(as
, CC_NE
);
760 emit_cmpi(as
, type
, (int32_t)ir_knum(irkey
)->u32
.hi
);
762 if (ra_hasreg(key
)) {
763 emit_cmpi(as
, key
, irkey
->i
); /* May use RID_TMP, i.e. type. */
764 asm_guardcc(as
, CC_NE
);
766 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(irkey
->t
));
768 if (ra_hasreg(key
)) emit_tai(as
, PPCI_LWZ
, key
, idx
, kofs
+4);
769 emit_tai(as
, PPCI_LWZ
, type
, idx
, kofs
);
771 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (ofs
+ 32768) >> 16);
772 emit_tai(as
, PPCI_ADDI
, dest
, node
, ofs
);
776 static void asm_newref(ASMState
*as
, IRIns
*ir
)
778 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
780 if (ir
->r
== RID_SINK
)
782 args
[0] = ASMREF_L
; /* lua_State *L */
783 args
[1] = ir
->op1
; /* GCtab *t */
784 args
[2] = ASMREF_TMP1
; /* cTValue *key */
785 asm_setupresult(as
, ir
, ci
); /* TValue * */
786 asm_gencall(as
, ci
, args
);
787 asm_tvptr(as
, ra_releasetmp(as
, ASMREF_TMP1
), ir
->op2
);
790 static void asm_uref(ASMState
*as
, IRIns
*ir
)
792 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
793 if (irref_isk(ir
->op1
)) {
794 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
795 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
796 emit_lsptr(as
, PPCI_LWZ
, dest
, v
, RSET_GPR
);
798 Reg uv
= ra_scratch(as
, RSET_GPR
);
799 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
800 if (ir
->o
== IR_UREFC
) {
801 asm_guardcc(as
, CC_NE
);
802 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, 1);
803 emit_tai(as
, PPCI_ADDI
, dest
, uv
, (int32_t)offsetof(GCupval
, tv
));
804 emit_tai(as
, PPCI_LBZ
, RID_TMP
, uv
, (int32_t)offsetof(GCupval
, closed
));
806 emit_tai(as
, PPCI_LWZ
, dest
, uv
, (int32_t)offsetof(GCupval
, v
));
808 emit_tai(as
, PPCI_LWZ
, uv
, func
,
809 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
813 static void asm_fref(ASMState
*as
, IRIns
*ir
)
815 UNUSED(as
); UNUSED(ir
);
816 lua_assert(!ra_used(ir
));
819 static void asm_strref(ASMState
*as
, IRIns
*ir
)
821 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
822 IRRef ref
= ir
->op2
, refk
= ir
->op1
;
823 int32_t ofs
= (int32_t)sizeof(GCstr
);
825 if (irref_isk(ref
)) {
826 IRRef tmp
= refk
; refk
= ref
; ref
= tmp
;
827 } else if (!irref_isk(refk
)) {
828 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
829 IRIns
*irr
= IR(ir
->op2
);
830 if (ra_hasreg(irr
->r
)) {
831 ra_noweak(as
, irr
->r
);
833 } else if (mayfuse(as
, irr
->op2
) &&
834 irr
->o
== IR_ADD
&& irref_isk(irr
->op2
) &&
835 checki16(ofs
+ IR(irr
->op2
)->i
)) {
836 ofs
+= IR(irr
->op2
)->i
;
837 right
= ra_alloc1(as
, irr
->op1
, rset_exclude(RSET_GPR
, left
));
839 right
= ra_allocref(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
841 emit_tai(as
, PPCI_ADDI
, dest
, dest
, ofs
);
842 emit_tab(as
, PPCI_ADD
, dest
, left
, right
);
845 r
= ra_alloc1(as
, ref
, RSET_GPR
);
848 emit_tai(as
, PPCI_ADDI
, dest
, r
, ofs
);
850 emit_tab(as
, PPCI_ADD
, dest
, r
,
851 ra_allock(as
, ofs
, rset_exclude(RSET_GPR
, r
)));
854 /* -- Loads and stores ---------------------------------------------------- */
856 static PPCIns
asm_fxloadins(IRIns
*ir
)
858 switch (irt_type(ir
->t
)) {
859 case IRT_I8
: return PPCI_LBZ
; /* Needs sign-extension. */
860 case IRT_U8
: return PPCI_LBZ
;
861 case IRT_I16
: return PPCI_LHA
;
862 case IRT_U16
: return PPCI_LHZ
;
863 case IRT_NUM
: return PPCI_LFD
;
864 case IRT_FLOAT
: return PPCI_LFS
;
865 default: return PPCI_LWZ
;
869 static PPCIns
asm_fxstoreins(IRIns
*ir
)
871 switch (irt_type(ir
->t
)) {
872 case IRT_I8
: case IRT_U8
: return PPCI_STB
;
873 case IRT_I16
: case IRT_U16
: return PPCI_STH
;
874 case IRT_NUM
: return PPCI_STFD
;
875 case IRT_FLOAT
: return PPCI_STFS
;
876 default: return PPCI_STW
;
880 static void asm_fload(ASMState
*as
, IRIns
*ir
)
882 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
883 Reg idx
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
884 PPCIns pi
= asm_fxloadins(ir
);
886 if (ir
->op2
== IRFL_TAB_ARRAY
) {
887 ofs
= asm_fuseabase(as
, ir
->op1
);
888 if (ofs
) { /* Turn the t->array load into an add for colocated arrays. */
889 emit_tai(as
, PPCI_ADDI
, dest
, idx
, ofs
);
893 ofs
= field_ofs
[ir
->op2
];
894 lua_assert(!irt_isi8(ir
->t
));
895 emit_tai(as
, pi
, dest
, idx
, ofs
);
898 static void asm_fstore(ASMState
*as
, IRIns
*ir
)
900 if (ir
->r
!= RID_SINK
) {
901 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
902 IRIns
*irf
= IR(ir
->op1
);
903 Reg idx
= ra_alloc1(as
, irf
->op1
, rset_exclude(RSET_GPR
, src
));
904 int32_t ofs
= field_ofs
[irf
->op2
];
905 PPCIns pi
= asm_fxstoreins(ir
);
906 emit_tai(as
, pi
, src
, idx
, ofs
);
910 static void asm_xload(ASMState
*as
, IRIns
*ir
)
912 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
913 lua_assert(!(ir
->op2
& IRXLOAD_UNALIGNED
));
915 emit_as(as
, PPCI_EXTSB
, dest
, dest
);
916 asm_fusexref(as
, asm_fxloadins(ir
), dest
, ir
->op1
, RSET_GPR
, 0);
919 static void asm_xstore(ASMState
*as
, IRIns
*ir
, int32_t ofs
)
922 if (ir
->r
== RID_SINK
)
924 if (ofs
== 0 && mayfuse(as
, ir
->op2
) && (irb
= IR(ir
->op2
))->o
== IR_BSWAP
&&
925 ra_noreg(irb
->r
) && (irt_isint(ir
->t
) || irt_isu32(ir
->t
))) {
926 /* Fuse BSWAP with XSTORE to stwbrx. */
927 Reg src
= ra_alloc1(as
, irb
->op1
, RSET_GPR
);
928 asm_fusexrefx(as
, PPCI_STWBRX
, src
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
930 Reg src
= ra_alloc1(as
, ir
->op2
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
931 asm_fusexref(as
, asm_fxstoreins(ir
), src
, ir
->op1
,
932 rset_exclude(RSET_GPR
, src
), ofs
);
936 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
939 Reg dest
= RID_NONE
, type
= RID_TMP
, tmp
= RID_TMP
, idx
;
940 RegSet allow
= RSET_GPR
;
941 int32_t ofs
= AHUREF_LSX
;
943 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
944 if (!irt_isnum(t
)) ofs
= 0;
945 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
946 rset_clear(allow
, dest
);
948 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
950 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, rset_exclude(allow
, idx
));
951 asm_guardcc(as
, CC_GE
);
952 emit_ab(as
, PPCI_CMPLW
, type
, tisnum
);
953 if (ra_hasreg(dest
)) {
954 if (ofs
== AHUREF_LSX
) {
955 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(RSET_GPR
,
956 (idx
&255)), (idx
>>8)));
957 emit_fab(as
, PPCI_LFDX
, dest
, (idx
&255), tmp
);
959 emit_fai(as
, PPCI_LFD
, dest
, idx
, ofs
);
963 asm_guardcc(as
, CC_NE
);
964 emit_ai(as
, PPCI_CMPWI
, type
, irt_toitype(t
));
965 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, idx
, ofs
+4);
967 if (ofs
== AHUREF_LSX
) {
968 emit_tab(as
, PPCI_LWZX
, type
, (idx
&255), tmp
);
969 emit_slwi(as
, tmp
, (idx
>>8), 3);
971 emit_tai(as
, PPCI_LWZ
, type
, idx
, ofs
);
975 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
977 RegSet allow
= RSET_GPR
;
978 Reg idx
, src
= RID_NONE
, type
= RID_NONE
;
979 int32_t ofs
= AHUREF_LSX
;
980 if (ir
->r
== RID_SINK
)
982 if (irt_isnum(ir
->t
)) {
983 src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
985 if (!irt_ispri(ir
->t
)) {
986 src
= ra_alloc1(as
, ir
->op2
, allow
);
987 rset_clear(allow
, src
);
990 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
991 rset_clear(allow
, type
);
993 idx
= asm_fuseahuref(as
, ir
->op1
, &ofs
, allow
);
994 if (irt_isnum(ir
->t
)) {
995 if (ofs
== AHUREF_LSX
) {
996 emit_fab(as
, PPCI_STFDX
, src
, (idx
&255), RID_TMP
);
997 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
999 emit_fai(as
, PPCI_STFD
, src
, idx
, ofs
);
1003 emit_tai(as
, PPCI_STW
, src
, idx
, ofs
+4);
1004 if (ofs
== AHUREF_LSX
) {
1005 emit_tab(as
, PPCI_STWX
, type
, (idx
&255), RID_TMP
);
1006 emit_slwi(as
, RID_TMP
, (idx
>>8), 3);
1008 emit_tai(as
, PPCI_STW
, type
, idx
, ofs
);
1013 static void asm_sload(ASMState
*as
, IRIns
*ir
)
1015 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 0 : 4);
1017 Reg dest
= RID_NONE
, type
= RID_NONE
, base
;
1018 RegSet allow
= RSET_GPR
;
1019 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
1020 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
1021 lua_assert(LJ_DUALNUM
||
1022 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
1023 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
1024 dest
= ra_scratch(as
, RSET_FPR
);
1025 asm_tointg(as
, ir
, dest
);
1026 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
1027 } else if (ra_used(ir
)) {
1028 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
1029 dest
= ra_dest(as
, ir
, irt_isnum(t
) ? RSET_FPR
: RSET_GPR
);
1030 rset_clear(allow
, dest
);
1031 base
= ra_alloc1(as
, REF_BASE
, allow
);
1032 rset_clear(allow
, base
);
1033 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1035 emit_tai(as
, PPCI_LWZ
, dest
, RID_SP
, SPOFS_TMPLO
);
1036 dest
= ra_scratch(as
, RSET_FPR
);
1037 emit_fai(as
, PPCI_STFD
, dest
, RID_SP
, SPOFS_TMP
);
1038 emit_fb(as
, PPCI_FCTIWZ
, dest
, dest
);
1039 t
.irt
= IRT_NUM
; /* Check for original type. */
1041 Reg tmp
= ra_scratch(as
, allow
);
1042 Reg hibias
= ra_allock(as
, 0x43300000, rset_clear(allow
, tmp
));
1043 Reg fbias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
1044 emit_fab(as
, PPCI_FSUB
, dest
, dest
, fbias
);
1045 emit_fai(as
, PPCI_LFD
, dest
, RID_SP
, SPOFS_TMP
);
1046 emit_lsptr(as
, PPCI_LFS
, (fbias
& 31),
1047 (void *)lj_ir_k64_find(as
->J
, U64x(59800004,59800000)),
1048 rset_clear(allow
, hibias
));
1049 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPLO
);
1050 emit_tai(as
, PPCI_STW
, hibias
, RID_SP
, SPOFS_TMPHI
);
1051 emit_asi(as
, PPCI_XORIS
, tmp
, tmp
, 0x8000);
1053 t
.irt
= IRT_INT
; /* Check for original type. */
1058 base
= ra_alloc1(as
, REF_BASE
, allow
);
1059 rset_clear(allow
, base
);
1062 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1063 Reg tisnum
= ra_allock(as
, (int32_t)LJ_TISNUM
, allow
);
1064 asm_guardcc(as
, CC_GE
);
1065 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tisnum
);
1068 if (ra_hasreg(dest
)) emit_fai(as
, PPCI_LFD
, dest
, base
, ofs
-4);
1070 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1071 asm_guardcc(as
, CC_NE
);
1072 emit_ai(as
, PPCI_CMPWI
, RID_TMP
, irt_toitype(t
));
1075 if (ra_hasreg(dest
)) emit_tai(as
, PPCI_LWZ
, dest
, base
, ofs
);
1077 if (ra_hasreg(type
)) emit_tai(as
, PPCI_LWZ
, type
, base
, ofs
-4);
1080 /* -- Allocations --------------------------------------------------------- */
1083 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1085 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1086 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1087 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1088 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1089 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1091 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1092 RegSet drop
= RSET_SCRATCH
;
1093 lua_assert(sz
!= CTSIZE_INVALID
);
1095 args
[0] = ASMREF_L
; /* lua_State *L */
1096 args
[1] = ASMREF_TMP1
; /* MSize size */
1099 if (ra_hasreg(ir
->r
))
1100 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1101 ra_evictset(as
, drop
);
1103 ra_destreg(as
, ir
, RID_RET
); /* GCcdata * */
1105 /* Initialize immutable cdata object. */
1106 if (ir
->o
== IR_CNEWI
) {
1107 int32_t ofs
= sizeof(GCcdata
);
1108 lua_assert(sz
== 4 || sz
== 8);
1111 lua_assert((ir
+1)->o
== IR_HIOP
);
1114 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1115 emit_tai(as
, PPCI_STW
, r
, RID_RET
, ofs
);
1116 rset_clear(allow
, r
);
1117 if (ofs
== sizeof(GCcdata
)) break;
1121 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1122 emit_tai(as
, PPCI_STB
, RID_RET
+1, RID_RET
, offsetof(GCcdata
, gct
));
1123 emit_tai(as
, PPCI_STH
, RID_TMP
, RID_RET
, offsetof(GCcdata
, ctypeid
));
1124 emit_ti(as
, PPCI_LI
, RID_RET
+1, ~LJ_TCDATA
);
1125 emit_ti(as
, PPCI_LI
, RID_TMP
, ctypeid
); /* Lower 16 bit used. Sign-ext ok. */
1126 asm_gencall(as
, ci
, args
);
1127 ra_allockreg(as
, (int32_t)(sz
+sizeof(GCcdata
)),
1128 ra_releasetmp(as
, ASMREF_TMP1
));
1131 #define asm_cnew(as, ir) ((void)0)
1134 /* -- Write barriers ------------------------------------------------------ */
1136 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1138 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1139 Reg mark
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1141 MCLabel l_end
= emit_label(as
);
1142 emit_tai(as
, PPCI_STW
, link
, tab
, (int32_t)offsetof(GCtab
, gclist
));
1143 emit_tai(as
, PPCI_STB
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1144 emit_setgl(as
, tab
, gc
.grayagain
);
1145 lua_assert(LJ_GC_BLACK
== 0x04);
1146 emit_rot(as
, PPCI_RLWINM
, mark
, mark
, 0, 30, 28); /* Clear black bit. */
1147 emit_getgl(as
, link
, gc
.grayagain
);
1148 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1149 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, mark
, LJ_GC_BLACK
);
1150 emit_tai(as
, PPCI_LBZ
, mark
, tab
, (int32_t)offsetof(GCtab
, marked
));
1153 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1155 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1159 /* No need for other object barriers (yet). */
1160 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1161 ra_evictset(as
, RSET_SCRATCH
);
1162 l_end
= emit_label(as
);
1163 args
[0] = ASMREF_TMP1
; /* global_State *g */
1164 args
[1] = ir
->op1
; /* TValue *tv */
1165 asm_gencall(as
, ci
, args
);
1166 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1167 obj
= IR(ir
->op1
)->r
;
1168 tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, obj
));
1169 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_EQ
, l_end
);
1170 emit_asi(as
, PPCI_ANDIDOT
, tmp
, tmp
, LJ_GC_BLACK
);
1171 emit_condbranch(as
, PPCI_BC
, CC_EQ
, l_end
);
1172 emit_asi(as
, PPCI_ANDIDOT
, RID_TMP
, RID_TMP
, LJ_GC_WHITES
);
1173 val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, obj
));
1174 emit_tai(as
, PPCI_LBZ
, tmp
, obj
,
1175 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1176 emit_tai(as
, PPCI_LBZ
, RID_TMP
, val
, (int32_t)offsetof(GChead
, marked
));
1179 /* -- Arithmetic and logic operations ------------------------------------- */
1181 static void asm_fparith(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1183 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1184 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1185 right
= (left
>> 8); left
&= 255;
1186 if (pi
== PPCI_FMUL
)
1187 emit_fac(as
, pi
, dest
, left
, right
);
1189 emit_fab(as
, pi
, dest
, left
, right
);
1192 static void asm_fpunary(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1194 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1195 Reg left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_FPR
);
1196 emit_fb(as
, pi
, dest
, left
);
1199 static int asm_fpjoin_pow(ASMState
*as
, IRIns
*ir
)
1201 IRIns
*irp
= IR(ir
->op1
);
1202 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1203 IRIns
*irpp
= IR(irp
->op1
);
1204 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1205 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1206 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_pow
];
1208 args
[0] = irpp
->op1
;
1210 asm_setupresult(as
, ir
, ci
);
1211 asm_gencall(as
, ci
, args
);
1218 static void asm_add(ASMState
*as
, IRIns
*ir
)
1220 if (irt_isnum(ir
->t
)) {
1221 if (!asm_fusemadd(as
, ir
, PPCI_FMADD
, PPCI_FMADD
))
1222 asm_fparith(as
, ir
, PPCI_FADD
);
1224 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1225 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1227 if (irref_isk(ir
->op2
)) {
1228 int32_t k
= IR(ir
->op2
)->i
;
1231 /* May fail due to spills/restores above, but simplifies the logic. */
1232 if (as
->flagmcp
== as
->mcp
) {
1237 emit_tai(as
, pi
, dest
, left
, k
);
1239 } else if ((k
& 0xffff) == 0) {
1240 emit_tai(as
, PPCI_ADDIS
, dest
, left
, (k
>> 16));
1242 } else if (!as
->sectref
) {
1243 emit_tai(as
, PPCI_ADDIS
, dest
, dest
, (k
+ 32768) >> 16);
1244 emit_tai(as
, PPCI_ADDI
, dest
, left
, k
);
1249 /* May fail due to spills/restores above, but simplifies the logic. */
1250 if (as
->flagmcp
== as
->mcp
) {
1255 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1256 emit_tab(as
, pi
, dest
, left
, right
);
1260 static void asm_sub(ASMState
*as
, IRIns
*ir
)
1262 if (irt_isnum(ir
->t
)) {
1263 if (!asm_fusemadd(as
, ir
, PPCI_FMSUB
, PPCI_FNMSUB
))
1264 asm_fparith(as
, ir
, PPCI_FSUB
);
1266 PPCIns pi
= PPCI_SUBF
;
1267 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1269 if (irref_isk(ir
->op1
)) {
1270 int32_t k
= IR(ir
->op1
)->i
;
1272 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1273 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1277 /* May fail due to spills/restores above, but simplifies the logic. */
1278 if (as
->flagmcp
== as
->mcp
) {
1283 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1284 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1285 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1289 static void asm_mul(ASMState
*as
, IRIns
*ir
)
1291 if (irt_isnum(ir
->t
)) {
1292 asm_fparith(as
, ir
, PPCI_FMUL
);
1294 PPCIns pi
= PPCI_MULLW
;
1295 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1296 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1297 if (irref_isk(ir
->op2
)) {
1298 int32_t k
= IR(ir
->op2
)->i
;
1300 emit_tai(as
, PPCI_MULLI
, dest
, left
, k
);
1304 /* May fail due to spills/restores above, but simplifies the logic. */
1305 if (as
->flagmcp
== as
->mcp
) {
1310 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1311 emit_tab(as
, pi
, dest
, left
, right
);
1315 static void asm_neg(ASMState
*as
, IRIns
*ir
)
1317 if (irt_isnum(ir
->t
)) {
1318 asm_fpunary(as
, ir
, PPCI_FNEG
);
1321 PPCIns pi
= PPCI_NEG
;
1322 if (as
->flagmcp
== as
->mcp
) {
1327 dest
= ra_dest(as
, ir
, RSET_GPR
);
1328 left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1329 emit_tab(as
, pi
, dest
, left
, 0);
1333 static void asm_arithov(ASMState
*as
, IRIns
*ir
, PPCIns pi
)
1335 Reg dest
, left
, right
;
1336 if (as
->flagmcp
== as
->mcp
) {
1340 asm_guardcc(as
, CC_SO
);
1341 dest
= ra_dest(as
, ir
, RSET_GPR
);
1342 left
= ra_alloc2(as
, ir
, RSET_GPR
);
1343 right
= (left
>> 8); left
&= 255;
1344 if (pi
== PPCI_SUBFO
) { Reg tmp
= left
; left
= right
; right
= tmp
; }
1345 emit_tab(as
, pi
|PPCF_DOT
, dest
, left
, right
);
1349 static void asm_add64(ASMState
*as
, IRIns
*ir
)
1351 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1352 Reg right
, left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1353 PPCIns pi
= PPCI_ADDE
;
1354 if (irref_isk(ir
->op2
)) {
1355 int32_t k
= IR(ir
->op2
)->i
;
1365 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1367 emit_tab(as
, pi
, dest
, left
, right
);
1369 dest
= ra_dest(as
, ir
, RSET_GPR
);
1370 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1371 if (irref_isk(ir
->op2
)) {
1372 int32_t k
= IR(ir
->op2
)->i
;
1374 emit_tai(as
, PPCI_ADDIC
, dest
, left
, k
);
1378 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1379 emit_tab(as
, PPCI_ADDC
, dest
, left
, right
);
1382 static void asm_sub64(ASMState
*as
, IRIns
*ir
)
1384 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1385 Reg left
, right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1386 PPCIns pi
= PPCI_SUBFE
;
1387 if (irref_isk(ir
->op1
)) {
1388 int32_t k
= IR(ir
->op1
)->i
;
1398 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1400 emit_tab(as
, pi
, dest
, right
, left
); /* Subtract right _from_ left. */
1402 dest
= ra_dest(as
, ir
, RSET_GPR
);
1403 right
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1404 if (irref_isk(ir
->op1
)) {
1405 int32_t k
= IR(ir
->op1
)->i
;
1407 emit_tai(as
, PPCI_SUBFIC
, dest
, right
, k
);
1411 left
= ra_alloc1(as
, ir
->op1
, rset_exclude(RSET_GPR
, right
));
1412 emit_tab(as
, PPCI_SUBFC
, dest
, right
, left
);
1415 static void asm_neg64(ASMState
*as
, IRIns
*ir
)
1417 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1418 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1419 emit_tab(as
, PPCI_SUBFZE
, dest
, left
, 0);
1421 dest
= ra_dest(as
, ir
, RSET_GPR
);
1422 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1423 emit_tai(as
, PPCI_SUBFIC
, dest
, left
, 0);
1427 static void asm_bitnot(ASMState
*as
, IRIns
*ir
)
1429 Reg dest
, left
, right
;
1430 PPCIns pi
= PPCI_NOR
;
1431 if (as
->flagmcp
== as
->mcp
) {
1436 dest
= ra_dest(as
, ir
, RSET_GPR
);
1437 if (mayfuse(as
, ir
->op1
)) {
1438 IRIns
*irl
= IR(ir
->op1
);
1439 if (irl
->o
== IR_BAND
)
1440 pi
^= (PPCI_NOR
^ PPCI_NAND
);
1441 else if (irl
->o
== IR_BXOR
)
1442 pi
^= (PPCI_NOR
^ PPCI_EQV
);
1443 else if (irl
->o
!= IR_BOR
)
1445 left
= ra_hintalloc(as
, irl
->op1
, dest
, RSET_GPR
);
1446 right
= ra_alloc1(as
, irl
->op2
, rset_exclude(RSET_GPR
, left
));
1449 left
= right
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1451 emit_asb(as
, pi
, dest
, left
, right
);
1454 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1456 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1458 if (mayfuse(as
, ir
->op1
) && (irx
= IR(ir
->op1
))->o
== IR_XLOAD
&&
1459 ra_noreg(irx
->r
) && (irt_isint(irx
->t
) || irt_isu32(irx
->t
))) {
1460 /* Fuse BSWAP with XLOAD to lwbrx. */
1461 asm_fusexrefx(as
, PPCI_LWBRX
, dest
, irx
->op1
, RSET_GPR
);
1463 Reg left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1467 emit_mr(as
, dest
, RID_TMP
);
1469 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 16, 23);
1470 emit_rot(as
, PPCI_RLWIMI
, tmp
, left
, 24, 0, 7);
1471 emit_rotlwi(as
, tmp
, left
, 8);
1475 static void asm_bitop(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1477 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1478 Reg right
, left
= ra_hintalloc(as
, ir
->op1
, dest
, RSET_GPR
);
1479 if (irref_isk(ir
->op2
)) {
1480 int32_t k
= IR(ir
->op2
)->i
;
1482 if ((checku16(k
) || (k
& 0xffff) == 0) || (tmp
= dest
, !as
->sectref
)) {
1484 emit_asi(as
, pik
^ (PPCI_ORI
^ PPCI_ORIS
), dest
, tmp
, (k
>> 16));
1485 if ((k
& 0xffff) == 0) return;
1487 emit_asi(as
, pik
, dest
, left
, k
);
1491 /* May fail due to spills/restores above, but simplifies the logic. */
1492 if (as
->flagmcp
== as
->mcp
) {
1497 right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1498 emit_asb(as
, pi
, dest
, left
, right
);
1501 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1502 static void asm_fuseandsh(ASMState
*as
, PPCIns pi
, int32_t mask
, IRRef ref
)
1506 if (mayfuse(as
, ref
) && (ir
= IR(ref
), ra_noreg(ir
->r
)) &&
1507 irref_isk(ir
->op2
) && ir
->o
>= IR_BSHL
&& ir
->o
<= IR_BROR
) {
1508 int32_t sh
= (IR(ir
->op2
)->i
& 31);
1511 if ((mask
& ((1u<<sh
)-1))) goto nofuse
;
1514 if ((mask
& ~((~0u)>>sh
))) goto nofuse
;
1522 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1523 *--as
->mcp
= pi
| PPCF_T(left
) | PPCF_B(sh
);
1527 left
= ra_alloc1(as
, ref
, RSET_GPR
);
1528 *--as
->mcp
= pi
| PPCF_T(left
);
1531 static void asm_bitand(ASMState
*as
, IRIns
*ir
)
1533 Reg dest
, left
, right
;
1534 IRRef lref
= ir
->op1
;
1537 if (as
->flagmcp
== as
->mcp
) {
1542 dest
= ra_dest(as
, ir
, RSET_GPR
);
1543 if (irref_isk(ir
->op2
)) {
1544 int32_t k
= IR(ir
->op2
)->i
;
1546 /* First check for a contiguous bitmask as used by rlwinm. */
1547 uint32_t s1
= lj_ffs((uint32_t)k
);
1548 uint32_t k1
= ((uint32_t)k
>> s1
);
1549 if ((k1
& (k1
+1)) == 0) {
1550 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1551 PPCF_MB(31-lj_fls((uint32_t)k
)) | PPCF_ME(31-s1
),
1556 uint32_t s2
= lj_ffs(~(uint32_t)k
);
1557 uint32_t k2
= (~(uint32_t)k
>> s2
);
1558 if ((k2
& (k2
+1)) == 0) {
1559 asm_fuseandsh(as
, PPCI_RLWINM
|dot
| PPCF_A(dest
) |
1560 PPCF_MB(32-s2
) | PPCF_ME(30-lj_fls(~(uint32_t)k
)),
1567 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1568 emit_asi(as
, PPCI_ANDIDOT
, dest
, left
, k
);
1570 } else if ((k
& 0xffff) == 0) {
1571 left
= ra_alloc1(as
, lref
, RSET_GPR
);
1572 emit_asi(as
, PPCI_ANDISDOT
, dest
, left
, (k
>> 16));
1577 if (mayfuse(as
, op2
) && IR(op2
)->o
== IR_BNOT
&& ra_noreg(IR(op2
)->r
)) {
1578 dot
^= (PPCI_AND
^ PPCI_ANDC
);
1581 left
= ra_hintalloc(as
, lref
, dest
, RSET_GPR
);
1582 right
= ra_alloc1(as
, op2
, rset_exclude(RSET_GPR
, left
));
1583 emit_asb(as
, PPCI_AND
^ dot
, dest
, left
, right
);
1586 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, PPCIns pi
, PPCIns pik
)
1590 if (as
->flagmcp
== as
->mcp
) {
1595 dest
= ra_dest(as
, ir
, RSET_GPR
);
1596 left
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1597 if (irref_isk(ir
->op2
)) { /* Constant shifts. */
1598 int32_t shift
= (IR(ir
->op2
)->i
& 31);
1599 if (pik
== 0) /* SLWI */
1600 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, shift
, 0, 31-shift
);
1601 else if (pik
== 1) /* SRWI */
1602 emit_rot(as
, PPCI_RLWINM
|dot
, dest
, left
, (32-shift
)&31, shift
, 31);
1604 emit_asb(as
, pik
|dot
, dest
, left
, shift
);
1606 Reg right
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, left
));
1607 emit_asb(as
, pi
|dot
, dest
, left
, right
);
1611 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int ismax
)
1613 if (irt_isnum(ir
->t
)) {
1614 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1616 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1617 right
= (left
>> 8); left
&= 255;
1618 if (tmp
== left
|| tmp
== right
)
1619 tmp
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_FPR
,
1620 dest
), left
), right
));
1621 emit_facb(as
, PPCI_FSEL
, dest
, tmp
,
1622 ismax
? left
: right
, ismax
? right
: left
);
1623 emit_fab(as
, PPCI_FSUB
, tmp
, left
, right
);
1625 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1626 Reg tmp1
= RID_TMP
, tmp2
= dest
;
1627 Reg right
, left
= ra_alloc2(as
, ir
, RSET_GPR
);
1628 right
= (left
>> 8); left
&= 255;
1629 if (tmp2
== left
|| tmp2
== right
)
1630 tmp2
= ra_scratch(as
, rset_exclude(rset_exclude(rset_exclude(RSET_GPR
,
1631 dest
), left
), right
));
1632 emit_tab(as
, PPCI_ADD
, dest
, tmp2
, right
);
1633 emit_asb(as
, ismax
? PPCI_ANDC
: PPCI_AND
, tmp2
, tmp2
, tmp1
);
1634 emit_tab(as
, PPCI_SUBFE
, tmp1
, tmp1
, tmp1
);
1635 emit_tab(as
, PPCI_SUBFC
, tmp2
, tmp2
, tmp1
);
1636 emit_asi(as
, PPCI_XORIS
, tmp2
, right
, 0x8000);
1637 emit_asi(as
, PPCI_XORIS
, tmp1
, left
, 0x8000);
1641 /* -- Comparisons --------------------------------------------------------- */
1643 #define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
1644 #define CC_TWO 0x80 /* Check two flags for FP comparison. */
1646 /* Map of comparisons to flags. ORDER IR. */
1647 static const uint8_t asm_compmap
[IR_ABC
+1] = {
1648 /* op int cc FP cc */
1649 /* LT */ CC_GE
+ (CC_GE
<<4),
1650 /* GE */ CC_LT
+ (CC_LE
<<4) + CC_TWO
,
1651 /* LE */ CC_GT
+ (CC_GE
<<4) + CC_TWO
,
1652 /* GT */ CC_LE
+ (CC_LE
<<4),
1653 /* ULT */ CC_GE
+ CC_UNSIGNED
+ (CC_GT
<<4) + CC_TWO
,
1654 /* UGE */ CC_LT
+ CC_UNSIGNED
+ (CC_LT
<<4),
1655 /* ULE */ CC_GT
+ CC_UNSIGNED
+ (CC_GT
<<4),
1656 /* UGT */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
,
1657 /* EQ */ CC_NE
+ (CC_NE
<<4),
1658 /* NE */ CC_EQ
+ (CC_EQ
<<4),
1659 /* ABC */ CC_LE
+ CC_UNSIGNED
+ (CC_LT
<<4) + CC_TWO
/* Same as UGT. */
1662 static void asm_intcomp_(ASMState
*as
, IRRef lref
, IRRef rref
, Reg cr
, PPCCC cc
)
1664 Reg right
, left
= ra_alloc1(as
, lref
, RSET_GPR
);
1665 if (irref_isk(rref
)) {
1666 int32_t k
= IR(rref
)->i
;
1667 if ((cc
& CC_UNSIGNED
) == 0) { /* Signed comparison with constant. */
1669 emit_tai(as
, PPCI_CMPWI
, cr
, left
, k
);
1670 /* Signed comparison with zero and referencing previous ins? */
1671 if (k
== 0 && lref
== as
->curins
-1)
1672 as
->flagmcp
= as
->mcp
; /* Allow elimination of the compare. */
1674 } else if ((cc
& 3) == (CC_EQ
& 3)) { /* Use CMPLWI for EQ or NE. */
1676 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1678 } else if (!as
->sectref
&& ra_noreg(IR(rref
)->r
)) {
1679 emit_tai(as
, PPCI_CMPLWI
, cr
, RID_TMP
, k
);
1680 emit_asi(as
, PPCI_XORIS
, RID_TMP
, left
, (k
>> 16));
1684 } else { /* Unsigned comparison with constant. */
1686 emit_tai(as
, PPCI_CMPLWI
, cr
, left
, k
);
1691 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, left
));
1692 emit_tab(as
, (cc
& CC_UNSIGNED
) ? PPCI_CMPLW
: PPCI_CMPW
, cr
, left
, right
);
1695 static void asm_comp(ASMState
*as
, IRIns
*ir
)
1697 PPCCC cc
= asm_compmap
[ir
->o
];
1698 if (irt_isnum(ir
->t
)) {
1699 Reg right
, left
= ra_alloc2(as
, ir
, RSET_FPR
);
1700 right
= (left
>> 8); left
&= 255;
1701 asm_guardcc(as
, (cc
>> 4));
1703 emit_tab(as
, PPCI_CROR
, ((cc
>>4)&3), ((cc
>>4)&3), (CC_EQ
&3));
1704 emit_fab(as
, PPCI_FCMPU
, 0, left
, right
);
1706 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1707 if (irref_isk(lref
) && !irref_isk(rref
)) {
1708 /* Swap constants to the right (only for ABC). */
1709 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1710 if ((cc
& 2) == 0) cc
^= 1; /* LT <-> GT, LE <-> GE */
1712 asm_guardcc(as
, cc
);
1713 asm_intcomp_(as
, lref
, rref
, 0, cc
);
1718 /* 64 bit integer comparisons. */
1719 static void asm_comp64(ASMState
*as
, IRIns
*ir
)
1721 PPCCC cc
= asm_compmap
[(ir
-1)->o
];
1722 if ((cc
&3) == (CC_EQ
&3)) {
1723 asm_guardcc(as
, cc
);
1724 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CROR
,
1725 (CC_EQ
&3), (CC_EQ
&3), 4+(CC_EQ
&3));
1727 asm_guardcc(as
, CC_EQ
);
1728 emit_tab(as
, PPCI_CROR
, (CC_EQ
&3), (CC_EQ
&3), ((cc
^~(cc
>>2))&1));
1729 emit_tab(as
, (cc
&4) ? PPCI_CRAND
: PPCI_CRANDC
,
1730 (CC_EQ
&3), (CC_EQ
&3), 4+(cc
&3));
1732 /* Loword comparison sets cr1 and is unsigned, except for equality. */
1733 asm_intcomp_(as
, (ir
-1)->op1
, (ir
-1)->op2
, 4,
1734 cc
| ((cc
&3) == (CC_EQ
&3) ? 0 : CC_UNSIGNED
));
1735 /* Hiword comparison sets cr0. */
1736 asm_intcomp_(as
, ir
->op1
, ir
->op2
, 0, cc
);
1737 as
->flagmcp
= NULL
; /* Doesn't work here. */
1741 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1743 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1744 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
1747 /* HIOP is marked as a store because it needs its own DCE logic. */
1748 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
1749 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
1750 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
1751 as
->curins
--; /* Always skip the CONV. */
1755 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
1756 as
->curins
--; /* Always skip the loword comparison. */
1759 } else if ((ir
-1)->o
== IR_XSTORE
) {
1760 as
->curins
--; /* Handle both stores here. */
1761 if ((ir
-1)->r
!= RID_SINK
) {
1762 asm_xstore(as
, ir
, 0);
1763 asm_xstore(as
, ir
-1, 4);
1767 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
1768 switch ((ir
-1)->o
) {
1769 case IR_ADD
: as
->curins
--; asm_add64(as
, ir
); break;
1770 case IR_SUB
: as
->curins
--; asm_sub64(as
, ir
); break;
1771 case IR_NEG
: as
->curins
--; asm_neg64(as
, ir
); break;
1775 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
1778 /* Nothing to do here. Handled by lo op itself. */
1780 default: lua_assert(0); break;
1783 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused without FFI. */
1787 /* -- Stack handling ------------------------------------------------------ */
1789 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1790 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
1791 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
1793 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1794 Reg tmp
, pbase
= irp
? (ra_hasreg(irp
->r
) ? irp
->r
: RID_TMP
) : RID_BASE
;
1795 rset_clear(allow
, pbase
);
1796 tmp
= allow
? rset_pickbot(allow
) :
1797 (pbase
== RID_RETHI
? RID_RETLO
: RID_RETHI
);
1798 emit_condbranch(as
, PPCI_BC
, CC_LT
, asm_exitstub_addr(as
, exitno
));
1799 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
1800 emit_tai(as
, PPCI_LWZ
, tmp
, RID_SP
, SPOFS_TMPW
);
1802 ra_modified(as
, tmp
);
1803 emit_ai(as
, PPCI_CMPLWI
, RID_TMP
, (int32_t)(8*topslot
));
1804 emit_tab(as
, PPCI_SUBF
, RID_TMP
, pbase
, tmp
);
1805 emit_tai(as
, PPCI_LWZ
, tmp
, tmp
, offsetof(lua_State
, maxstack
));
1806 if (pbase
== RID_TMP
)
1807 emit_getgl(as
, RID_TMP
, jit_base
);
1808 emit_getgl(as
, tmp
, jit_L
);
1809 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
1810 emit_tai(as
, PPCI_STW
, tmp
, RID_SP
, SPOFS_TMPW
);
1813 /* Restore Lua stack from on-trace state. */
1814 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
1816 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
1817 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
1818 MSize n
, nent
= snap
->nent
;
1819 /* Store the value of all modified slots to the Lua stack. */
1820 for (n
= 0; n
< nent
; n
++) {
1821 SnapEntry sn
= map
[n
];
1822 BCReg s
= snap_slot(sn
);
1823 int32_t ofs
= 8*((int32_t)s
-1);
1824 IRRef ref
= snap_ref(sn
);
1825 IRIns
*ir
= IR(ref
);
1826 if ((sn
& SNAP_NORESTORE
))
1828 if (irt_isnum(ir
->t
)) {
1829 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
1830 emit_fai(as
, PPCI_STFD
, src
, RID_BASE
, ofs
);
1833 RegSet allow
= rset_exclude(RSET_GPR
, RID_BASE
);
1834 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) || irt_isinteger(ir
->t
));
1835 if (!irt_ispri(ir
->t
)) {
1836 Reg src
= ra_alloc1(as
, ref
, allow
);
1837 rset_clear(allow
, src
);
1838 emit_tai(as
, PPCI_STW
, src
, RID_BASE
, ofs
+4);
1840 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
1841 if (s
== 0) continue; /* Do not overwrite link to previous frame. */
1842 type
= ra_allock(as
, (int32_t)(*flinks
--), allow
);
1844 type
= ra_allock(as
, (int32_t)irt_toitype(ir
->t
), allow
);
1846 emit_tai(as
, PPCI_STW
, type
, RID_BASE
, ofs
);
1850 lua_assert(map
+ nent
== flinks
);
1853 /* -- GC handling --------------------------------------------------------- */
1855 /* Check GC threshold and do one or more GC steps. */
1856 static void asm_gc_check(ASMState
*as
)
1858 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
1862 ra_evictset(as
, RSET_SCRATCH
);
1863 l_end
= emit_label(as
);
1864 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1865 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
1866 emit_ai(as
, PPCI_CMPWI
, RID_RET
, 0);
1867 args
[0] = ASMREF_TMP1
; /* global_State *g */
1868 args
[1] = ASMREF_TMP2
; /* MSize steps */
1869 asm_gencall(as
, ci
, args
);
1870 emit_tai(as
, PPCI_ADDI
, ra_releasetmp(as
, ASMREF_TMP1
), RID_JGL
, -32768);
1871 tmp
= ra_releasetmp(as
, ASMREF_TMP2
);
1872 emit_loadi(as
, tmp
, as
->gcsteps
);
1873 /* Jump around GC step if GC total < GC threshold. */
1874 emit_condbranch(as
, PPCI_BC
|PPCF_Y
, CC_LT
, l_end
);
1875 emit_ab(as
, PPCI_CMPLW
, RID_TMP
, tmp
);
1876 emit_getgl(as
, tmp
, gc
.threshold
);
1877 emit_getgl(as
, RID_TMP
, gc
.total
);
1882 /* -- Loop handling ------------------------------------------------------- */
1884 /* Fixup the loop branch. */
1885 static void asm_loop_fixup(ASMState
*as
)
1887 MCode
*p
= as
->mctop
;
1888 MCode
*target
= as
->mcp
;
1889 if (as
->loopinv
) { /* Inverted loop branch? */
1890 /* asm_guardcc already inverted the cond branch and patched the final b. */
1891 p
[-2] = (p
[-2] & (0xffff0000u
& ~PPCF_Y
)) | (((target
-p
+2) & 0x3fffu
) << 2);
1893 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1897 /* -- Head of trace ------------------------------------------------------- */
1899 /* Coalesce BASE register for a root trace. */
1900 static void asm_head_root_base(ASMState
*as
)
1902 IRIns
*ir
= IR(REF_BASE
);
1906 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1907 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1909 emit_mr(as
, r
, RID_BASE
);
1913 /* Coalesce BASE register for a side trace. */
1914 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
1916 IRIns
*ir
= IR(REF_BASE
);
1920 if (rset_test(as
->modset
, r
) || irt_ismarked(ir
->t
))
1921 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
1923 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
1924 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
1925 rset_clear(allow
, irp
->r
);
1926 emit_mr(as
, r
, irp
->r
); /* Move from coalesced parent reg. */
1928 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
1934 /* -- Tail of trace ------------------------------------------------------- */
1936 /* Fixup the tail code. */
1937 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
1939 MCode
*p
= as
->mctop
;
1941 int32_t spadj
= as
->T
->spadjust
;
1947 /* Patch stack adjustment. */
1948 lua_assert(checki16(CFRAME_SIZE
+spadj
));
1949 p
[-3] = PPCI_ADDI
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | (CFRAME_SIZE
+spadj
);
1950 p
[-2] = PPCI_STWU
| PPCF_T(RID_TMP
) | PPCF_A(RID_SP
) | spadj
;
1952 /* Patch exit branch. */
1953 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
1954 p
[-1] = PPCI_B
|(((target
-p
+1)&0x00ffffffu
)<<2);
1957 /* Prepare tail of code. */
1958 static void asm_tail_prep(ASMState
*as
)
1960 MCode
*p
= as
->mctop
- 1; /* Leave room for exit branch. */
1962 as
->invmcp
= as
->mcp
= p
;
1964 as
->mcp
= p
-2; /* Leave room for stack pointer adjustment. */
1969 /* -- Instruction dispatch ------------------------------------------------ */
1971 /* Assemble a single instruction. */
1972 static void asm_ir(ASMState
*as
, IRIns
*ir
)
1974 switch ((IROp
)ir
->o
) {
1975 /* Miscellaneous ops. */
1976 case IR_LOOP
: asm_loop(as
); break;
1977 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
1979 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
1980 case IR_PHI
: asm_phi(as
, ir
); break;
1981 case IR_HIOP
: asm_hiop(as
, ir
); break;
1982 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
1984 /* Guarded assertions. */
1985 case IR_EQ
: case IR_NE
:
1986 if ((ir
-1)->o
== IR_HREF
&& ir
->op1
== as
->curins
-1) {
1988 asm_href(as
, ir
-1, (IROp
)ir
->o
);
1992 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
1993 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
1998 case IR_RETF
: asm_retf(as
, ir
); break;
2001 case IR_BNOT
: asm_bitnot(as
, ir
); break;
2002 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
2004 case IR_BAND
: asm_bitand(as
, ir
); break;
2005 case IR_BOR
: asm_bitop(as
, ir
, PPCI_OR
, PPCI_ORI
); break;
2006 case IR_BXOR
: asm_bitop(as
, ir
, PPCI_XOR
, PPCI_XORI
); break;
2008 case IR_BSHL
: asm_bitshift(as
, ir
, PPCI_SLW
, 0); break;
2009 case IR_BSHR
: asm_bitshift(as
, ir
, PPCI_SRW
, 1); break;
2010 case IR_BSAR
: asm_bitshift(as
, ir
, PPCI_SRAW
, PPCI_SRAWI
); break;
2011 case IR_BROL
: asm_bitshift(as
, ir
, PPCI_RLWNM
|PPCF_MB(0)|PPCF_ME(31),
2012 PPCI_RLWINM
|PPCF_MB(0)|PPCF_ME(31)); break;
2013 case IR_BROR
: lua_assert(0); break;
2015 /* Arithmetic ops. */
2016 case IR_ADD
: asm_add(as
, ir
); break;
2017 case IR_SUB
: asm_sub(as
, ir
); break;
2018 case IR_MUL
: asm_mul(as
, ir
); break;
2019 case IR_DIV
: asm_fparith(as
, ir
, PPCI_FDIV
); break;
2020 case IR_MOD
: asm_callid(as
, ir
, IRCALL_lj_vm_modi
); break;
2021 case IR_POW
: asm_callid(as
, ir
, IRCALL_lj_vm_powi
); break;
2022 case IR_NEG
: asm_neg(as
, ir
); break;
2024 case IR_ABS
: asm_fpunary(as
, ir
, PPCI_FABS
); break;
2025 case IR_ATAN2
: asm_callid(as
, ir
, IRCALL_atan2
); break;
2026 case IR_LDEXP
: asm_callid(as
, ir
, IRCALL_ldexp
); break;
2027 case IR_MIN
: asm_min_max(as
, ir
, 0); break;
2028 case IR_MAX
: asm_min_max(as
, ir
, 1); break;
2030 if (ir
->op2
== IRFPM_EXP2
&& asm_fpjoin_pow(as
, ir
))
2032 if (ir
->op2
== IRFPM_SQRT
&& (as
->flags
& JIT_F_SQRT
))
2033 asm_fpunary(as
, ir
, PPCI_FSQRT
);
2035 asm_callid(as
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
2038 /* Overflow-checking arithmetic ops. */
2039 case IR_ADDOV
: asm_arithov(as
, ir
, PPCI_ADDO
); break;
2040 case IR_SUBOV
: asm_arithov(as
, ir
, PPCI_SUBFO
); break;
2041 case IR_MULOV
: asm_arithov(as
, ir
, PPCI_MULLWO
); break;
2043 /* Memory references. */
2044 case IR_AREF
: asm_aref(as
, ir
); break;
2045 case IR_HREF
: asm_href(as
, ir
, 0); break;
2046 case IR_HREFK
: asm_hrefk(as
, ir
); break;
2047 case IR_NEWREF
: asm_newref(as
, ir
); break;
2048 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
2049 case IR_FREF
: asm_fref(as
, ir
); break;
2050 case IR_STRREF
: asm_strref(as
, ir
); break;
2052 /* Loads and stores. */
2053 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
2054 asm_ahuvload(as
, ir
);
2056 case IR_FLOAD
: asm_fload(as
, ir
); break;
2057 case IR_XLOAD
: asm_xload(as
, ir
); break;
2058 case IR_SLOAD
: asm_sload(as
, ir
); break;
2060 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
2061 case IR_FSTORE
: asm_fstore(as
, ir
); break;
2062 case IR_XSTORE
: asm_xstore(as
, ir
, 0); break;
2065 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
2066 case IR_TNEW
: asm_tnew(as
, ir
); break;
2067 case IR_TDUP
: asm_tdup(as
, ir
); break;
2068 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
2070 /* Write barriers. */
2071 case IR_TBAR
: asm_tbar(as
, ir
); break;
2072 case IR_OBAR
: asm_obar(as
, ir
); break;
2074 /* Type conversions. */
2075 case IR_CONV
: asm_conv(as
, ir
); break;
2076 case IR_TOBIT
: asm_tobit(as
, ir
); break;
2077 case IR_TOSTR
: asm_tostr(as
, ir
); break;
2078 case IR_STRTO
: asm_strto(as
, ir
); break;
2081 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
2082 case IR_CALLXS
: asm_callx(as
, ir
); break;
2083 case IR_CARG
: break;
2086 setintV(&as
->J
->errinfo
, ir
->o
);
2087 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
2092 /* -- Trace setup --------------------------------------------------------- */
2094 /* Ensure there are enough stack slots for call arguments. */
2095 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
2097 IRRef args
[CCI_NARGS_MAX
*2];
2098 uint32_t i
, nargs
= (int)CCI_NARGS(ci
);
2099 int nslots
= 2, ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
2100 asm_collectargs(as
, ir
, ci
, args
);
2101 for (i
= 0; i
< nargs
; i
++)
2102 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
2103 if (nfpr
> 0) nfpr
--; else nslots
= (nslots
+3) & ~1;
2105 if (ngpr
> 0) ngpr
--; else nslots
++;
2107 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
2108 as
->evenspill
= nslots
;
2109 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
2112 static void asm_setup_target(ASMState
*as
)
2114 asm_exitstub_setup(as
, as
->T
->nsnap
+ (as
->parent
? 1 : 0));
2117 /* -- Trace patching ------------------------------------------------------ */
2119 /* Patch exit jumps of existing machine code to a new target. */
2120 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
2122 MCode
*p
= T
->mcode
;
2123 MCode
*pe
= (MCode
*)((char *)p
+ T
->szmcode
);
2124 MCode
*px
= exitstub_trace_addr(T
, exitno
);
2125 MCode
*cstart
= NULL
;
2126 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
2128 for (; p
< pe
; p
++) {
2129 /* Look for exitstub branch, try to replace with branch to target. */
2131 if ((ins
& 0xfc000000u
) == 0x40000000u
&&
2132 ((ins
^ ((char *)px
-(char *)p
)) & 0xffffu
) == 0) {
2133 ptrdiff_t delta
= (char *)target
- (char *)p
;
2134 if (((ins
>> 16) & 3) == (CC_SO
&3)) {
2135 clearso
= sizeof(MCode
);
2136 delta
-= sizeof(MCode
);
2138 /* Many, but not all short-range branches can be patched directly. */
2139 if (((delta
+ 0x8000) >> 16) == 0) {
2140 *p
= (ins
& 0xffdf0000u
) | ((uint32_t)delta
& 0xffffu
) |
2141 ((delta
& 0x8000) * (PPCF_Y
/0x8000));
2142 if (!cstart
) cstart
= p
;
2144 } else if ((ins
& 0xfc000000u
) == PPCI_B
&&
2145 ((ins
^ ((char *)px
-(char *)p
)) & 0x03ffffffu
) == 0) {
2146 ptrdiff_t delta
= (char *)target
- (char *)p
;
2147 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2148 *p
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2149 if (!cstart
) cstart
= p
;
2152 { /* Always patch long-range branch in exit stub itself. */
2153 ptrdiff_t delta
= (char *)target
- (char *)px
- clearso
;
2154 lua_assert(((delta
+ 0x02000000) >> 26) == 0);
2155 *px
= PPCI_B
| ((uint32_t)delta
& 0x03ffffffu
);
2157 if (!cstart
) cstart
= px
;
2158 lj_mcode_sync(cstart
, px
+1);
2159 if (clearso
) { /* Extend the current trace. Ugly workaround. */
2160 MCode
*pp
= J
->cur
.mcode
;
2161 J
->cur
.szmcode
+= sizeof(MCode
);
2162 *--pp
= PPCI_MCRXR
; /* Clear SO flag. */
2164 lj_mcode_sync(pp
, pp
+1);
2166 lj_mcode_patch(J
, mcarea
, 1);