2 ** x86/x64 IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Guard handling ------------------------------------------------------ */
8 /* Generate an exit stub group at the bottom of the reserved MCode memory. */
9 static MCode
*asm_exitstub_gen(ASMState
*as
, ExitNo group
)
11 ExitNo i
, groupofs
= (group
*EXITSTUBS_PER_GROUP
) & 0xff;
12 MCode
*mxp
= as
->mcbot
;
13 MCode
*mxpstart
= mxp
;
14 if (mxp
+ (2+2)*EXITSTUBS_PER_GROUP
+8+5 >= as
->mctop
)
16 /* Push low byte of exitno for each exit stub. */
17 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)groupofs
;
18 for (i
= 1; i
< EXITSTUBS_PER_GROUP
; i
++) {
19 *mxp
++ = XI_JMPs
; *mxp
++ = (MCode
)((2+2)*(EXITSTUBS_PER_GROUP
- i
) - 2);
20 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)(groupofs
+ i
);
22 /* Push the high byte of the exitno for each exit stub group. */
23 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)((group
*EXITSTUBS_PER_GROUP
)>>8);
24 /* Store DISPATCH at original stack slot 0. Account for the two push ops. */
26 *mxp
++ = MODRM(XM_OFS8
, 0, RID_ESP
);
27 *mxp
++ = MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
28 *mxp
++ = 2*sizeof(void *);
29 *(int32_t *)mxp
= ptr2addr(J2GG(as
->J
)->dispatch
); mxp
+= 4;
30 /* Jump to exit handler which fills in the ExitState. */
31 *mxp
++ = XI_JMP
; mxp
+= 4;
32 *((int32_t *)(mxp
-4)) = jmprel(mxp
, (MCode
*)(void *)lj_vm_exit_handler
);
33 /* Commit the code for this group (even if assembly fails later on). */
34 lj_mcode_commitbot(as
->J
, mxp
);
36 as
->mclim
= as
->mcbot
+ MCLIM_REDZONE
;
40 /* Setup all needed exit stubs. */
41 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
44 if (nexits
>= EXITSTUBS_PER_GROUP
*LJ_MAX_EXITSTUBGR
)
45 lj_trace_err(as
->J
, LJ_TRERR_SNAPOV
);
46 for (i
= 0; i
< (nexits
+EXITSTUBS_PER_GROUP
-1)/EXITSTUBS_PER_GROUP
; i
++)
47 if (as
->J
->exitstubgroup
[i
] == NULL
)
48 as
->J
->exitstubgroup
[i
] = asm_exitstub_gen(as
, i
);
51 /* Emit conditional branch to exit for guard.
52 ** It's important to emit this *after* all registers have been allocated,
53 ** because rematerializations may invalidate the flags.
55 static void asm_guardcc(ASMState
*as
, int cc
)
57 MCode
*target
= exitstub_addr(as
->J
, as
->snapno
);
59 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
61 *(int32_t *)(p
+1) = jmprel(p
+5, target
);
65 emit_sjcc(as
, cc
, target
);
69 emit_jcc(as
, cc
, target
);
72 /* -- Memory operand fusion ----------------------------------------------- */
74 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
75 #define CONFLICT_SEARCH_LIM 31
77 /* Check if a reference is a signed 32 bit constant. */
78 static int asm_isk32(ASMState
*as
, IRRef ref
, int32_t *k
)
82 if (ir
->o
!= IR_KINT64
) {
85 } else if (checki32((int64_t)ir_kint64(ir
)->u64
)) {
86 *k
= (int32_t)ir_kint64(ir
)->u64
;
93 /* Check if there's no conflicting instruction between curins and ref.
94 ** Also avoid fusing loads if there are multiple references.
96 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
, int noload
)
100 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
101 return 0; /* Give up, ref is too far away. */
103 if (ir
[i
].o
== conflict
)
104 return 0; /* Conflict found. */
105 else if (!noload
&& (ir
[i
].op1
== ref
|| ir
[i
].op2
== ref
))
108 return 1; /* Ok, no conflict. */
111 /* Fuse array base into memory operand. */
112 static IRRef
asm_fuseabase(ASMState
*as
, IRRef ref
)
114 IRIns
*irb
= IR(ref
);
116 if (irb
->o
== IR_FLOAD
) {
117 IRIns
*ira
= IR(irb
->op1
);
118 lua_assert(irb
->op2
== IRFL_TAB_ARRAY
);
119 /* We can avoid the FLOAD of t->array for colocated arrays. */
120 if (ira
->o
== IR_TNEW
&& ira
->op1
<= LJ_MAX_COLOSIZE
&&
121 !neverfuse(as
) && noconflict(as
, irb
->op1
, IR_NEWREF
, 1)) {
122 as
->mrm
.ofs
= (int32_t)sizeof(GCtab
); /* Ofs to colocated array. */
123 return irb
->op1
; /* Table obj. */
125 } else if (irb
->o
== IR_ADD
&& irref_isk(irb
->op2
)) {
126 /* Fuse base offset (vararg load). */
127 as
->mrm
.ofs
= IR(irb
->op2
)->i
;
130 return ref
; /* Otherwise use the given array base. */
133 /* Fuse array reference into memory operand. */
134 static void asm_fusearef(ASMState
*as
, IRIns
*ir
, RegSet allow
)
137 lua_assert(ir
->o
== IR_AREF
);
138 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, asm_fuseabase(as
, ir
->op1
), allow
);
140 if (irref_isk(ir
->op2
)) {
141 as
->mrm
.ofs
+= 8*irx
->i
;
142 as
->mrm
.idx
= RID_NONE
;
144 rset_clear(allow
, as
->mrm
.base
);
145 as
->mrm
.scale
= XM_SCALE8
;
146 /* Fuse a constant ADD (e.g. t[i+1]) into the offset.
147 ** Doesn't help much without ABCelim, but reduces register pressure.
149 if (!LJ_64
&& /* Has bad effects with negative index on x64. */
150 mayfuse(as
, ir
->op2
) && ra_noreg(irx
->r
) &&
151 irx
->o
== IR_ADD
&& irref_isk(irx
->op2
)) {
152 as
->mrm
.ofs
+= 8*IR(irx
->op2
)->i
;
153 as
->mrm
.idx
= (uint8_t)ra_alloc1(as
, irx
->op1
, allow
);
155 as
->mrm
.idx
= (uint8_t)ra_alloc1(as
, ir
->op2
, allow
);
160 /* Fuse array/hash/upvalue reference into memory operand.
161 ** Caveat: this may allocate GPRs for the base/idx registers. Be sure to
162 ** pass the final allow mask, excluding any GPRs used for other inputs.
163 ** In particular: 2-operand GPR instructions need to call ra_dest() first!
165 static void asm_fuseahuref(ASMState
*as
, IRRef ref
, RegSet allow
)
168 if (ra_noreg(ir
->r
)) {
169 switch ((IROp
)ir
->o
) {
171 if (mayfuse(as
, ref
)) {
172 asm_fusearef(as
, ir
, allow
);
177 if (mayfuse(as
, ref
)) {
178 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ir
->op1
, allow
);
179 as
->mrm
.ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
180 as
->mrm
.idx
= RID_NONE
;
185 if (irref_isk(ir
->op1
)) {
186 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
187 GCupval
*uv
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
;
188 as
->mrm
.ofs
= ptr2addr(&uv
->tv
);
189 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
194 lua_assert(ir
->o
== IR_HREF
|| ir
->o
== IR_NEWREF
|| ir
->o
== IR_UREFO
||
199 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ref
, allow
);
201 as
->mrm
.idx
= RID_NONE
;
204 /* Fuse FLOAD/FREF reference into memory operand. */
205 static void asm_fusefref(ASMState
*as
, IRIns
*ir
, RegSet allow
)
207 lua_assert(ir
->o
== IR_FLOAD
|| ir
->o
== IR_FREF
);
208 as
->mrm
.ofs
= field_ofs
[ir
->op2
];
209 as
->mrm
.idx
= RID_NONE
;
210 if (irref_isk(ir
->op1
)) {
211 as
->mrm
.ofs
+= IR(ir
->op1
)->i
;
212 as
->mrm
.base
= RID_NONE
;
214 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ir
->op1
, allow
);
218 /* Fuse string reference into memory operand. */
219 static void asm_fusestrref(ASMState
*as
, IRIns
*ir
, RegSet allow
)
222 lua_assert(ir
->o
== IR_STRREF
);
223 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
224 as
->mrm
.scale
= XM_SCALE1
;
225 as
->mrm
.ofs
= sizeof(GCstr
);
226 if (irref_isk(ir
->op1
)) {
227 as
->mrm
.ofs
+= IR(ir
->op1
)->i
;
229 Reg r
= ra_alloc1(as
, ir
->op1
, allow
);
230 rset_clear(allow
, r
);
231 as
->mrm
.base
= (uint8_t)r
;
234 if (irref_isk(ir
->op2
)) {
235 as
->mrm
.ofs
+= irr
->i
;
238 /* Fuse a constant add into the offset, e.g. string.sub(s, i+10). */
239 if (!LJ_64
&& /* Has bad effects with negative index on x64. */
240 mayfuse(as
, ir
->op2
) && irr
->o
== IR_ADD
&& irref_isk(irr
->op2
)) {
241 as
->mrm
.ofs
+= IR(irr
->op2
)->i
;
242 r
= ra_alloc1(as
, irr
->op1
, allow
);
244 r
= ra_alloc1(as
, ir
->op2
, allow
);
246 if (as
->mrm
.base
== RID_NONE
)
247 as
->mrm
.base
= (uint8_t)r
;
249 as
->mrm
.idx
= (uint8_t)r
;
253 static void asm_fusexref(ASMState
*as
, IRRef ref
, RegSet allow
)
256 as
->mrm
.idx
= RID_NONE
;
257 if (ir
->o
== IR_KPTR
|| ir
->o
== IR_KKPTR
) {
259 as
->mrm
.base
= RID_NONE
;
260 } else if (ir
->o
== IR_STRREF
) {
261 asm_fusestrref(as
, ir
, allow
);
264 if (canfuse(as
, ir
) && ir
->o
== IR_ADD
&& ra_noreg(ir
->r
)) {
265 /* Gather (base+idx*sz)+ofs as emitted by cdata ptr/array indexing. */
269 if (asm_isk32(as
, ir
->op2
, &as
->mrm
.ofs
)) { /* Recognize x+ofs. */
272 if (!(ir
->o
== IR_ADD
&& canfuse(as
, ir
) && ra_noreg(ir
->r
)))
275 as
->mrm
.scale
= XM_SCALE1
;
279 if (!(irx
->o
== IR_BSHL
|| irx
->o
== IR_ADD
)) { /* Try other operand. */
284 if (canfuse(as
, irx
) && ra_noreg(irx
->r
)) {
285 if (irx
->o
== IR_BSHL
&& irref_isk(irx
->op2
) && IR(irx
->op2
)->i
<= 3) {
286 /* Recognize idx<<b with b = 0-3, corresponding to sz = (1),2,4,8. */
288 as
->mrm
.scale
= (uint8_t)(IR(irx
->op2
)->i
<< 6);
289 } else if (irx
->o
== IR_ADD
&& irx
->op1
== irx
->op2
) {
290 /* FOLD does idx*2 ==> idx<<1 ==> idx+idx. */
292 as
->mrm
.scale
= XM_SCALE2
;
295 r
= ra_alloc1(as
, idx
, allow
);
296 rset_clear(allow
, r
);
297 as
->mrm
.idx
= (uint8_t)r
;
300 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ref
, allow
);
304 /* Fuse load into memory operand. */
305 static Reg
asm_fuseload(ASMState
*as
, IRRef ref
, RegSet allow
)
308 if (ra_hasreg(ir
->r
)) {
309 if (allow
!= RSET_EMPTY
) { /* Fast path. */
310 ra_noweak(as
, ir
->r
);
314 /* Force a spill if only memory operands are allowed (asm_x87load). */
315 as
->mrm
.base
= RID_ESP
;
316 as
->mrm
.ofs
= ra_spill(as
, ir
);
317 as
->mrm
.idx
= RID_NONE
;
320 if (ir
->o
== IR_KNUM
) {
321 RegSet avail
= as
->freeset
& ~as
->modset
& RSET_FPR
;
322 lua_assert(allow
!= RSET_EMPTY
);
323 if (!(avail
& (avail
-1))) { /* Fuse if less than two regs available. */
324 as
->mrm
.ofs
= ptr2addr(ir_knum(ir
));
325 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
328 } else if (mayfuse(as
, ref
)) {
329 RegSet xallow
= (allow
& RSET_GPR
) ? allow
: RSET_GPR
;
330 if (ir
->o
== IR_SLOAD
) {
331 if (!(ir
->op2
& (IRSLOAD_PARENT
|IRSLOAD_CONVERT
)) &&
332 noconflict(as
, ref
, IR_RETF
, 0)) {
333 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, REF_BASE
, xallow
);
334 as
->mrm
.ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
&IRSLOAD_FRAME
)?4:0);
335 as
->mrm
.idx
= RID_NONE
;
338 } else if (ir
->o
== IR_FLOAD
) {
339 /* Generic fusion is only ok for 32 bit operand (but see asm_comp). */
340 if ((irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
)) &&
341 noconflict(as
, ref
, IR_FSTORE
, 0)) {
342 asm_fusefref(as
, ir
, xallow
);
345 } else if (ir
->o
== IR_ALOAD
|| ir
->o
== IR_HLOAD
|| ir
->o
== IR_ULOAD
) {
346 if (noconflict(as
, ref
, ir
->o
+ IRDELTA_L2S
, 0)) {
347 asm_fuseahuref(as
, ir
->op1
, xallow
);
350 } else if (ir
->o
== IR_XLOAD
) {
351 /* Generic fusion is not ok for 8/16 bit operands (but see asm_comp).
352 ** Fusing unaligned memory operands is ok on x86 (except for SIMD types).
354 if ((!irt_typerange(ir
->t
, IRT_I8
, IRT_U16
)) &&
355 noconflict(as
, ref
, IR_XSTORE
, 0)) {
356 asm_fusexref(as
, ir
->op1
, xallow
);
359 } else if (ir
->o
== IR_VLOAD
) {
360 asm_fuseahuref(as
, ir
->op1
, xallow
);
364 if (!(as
->freeset
& allow
) &&
365 (allow
== RSET_EMPTY
|| ra_hasspill(ir
->s
) || iscrossref(as
, ref
)))
367 return ra_allocref(as
, ref
, allow
);
371 /* Don't fuse a 32 bit load into a 64 bit operation. */
372 static Reg
asm_fuseloadm(ASMState
*as
, IRRef ref
, RegSet allow
, int is64
)
374 if (is64
&& !irt_is64(IR(ref
)->t
))
375 return ra_alloc1(as
, ref
, allow
);
376 return asm_fuseload(as
, ref
, allow
);
379 #define asm_fuseloadm(as, ref, allow, is64) asm_fuseload(as, (ref), (allow))
382 /* -- Calls --------------------------------------------------------------- */
384 /* Count the required number of stack slots for a call. */
385 static int asm_count_call_slots(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
387 uint32_t i
, nargs
= CCI_NARGS(ci
);
391 nslots
= (int)(nargs
*2); /* Only matters for more than four args. */
393 int ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
394 for (i
= 0; i
< nargs
; i
++)
395 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
396 if (nfpr
> 0) nfpr
--; else nslots
+= 2;
398 if (ngpr
> 0) ngpr
--; else nslots
+= 2;
403 if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_FASTCALL
)
405 else if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_THISCALL
)
407 for (i
= 0; i
< nargs
; i
++)
408 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
409 nslots
+= irt_isnum(IR(args
[i
])->t
) ? 2 : 1;
411 if (ngpr
> 0) ngpr
--; else nslots
++;
417 /* Generate a call to a C function. */
418 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
420 uint32_t n
, nargs
= CCI_NARGS(ci
);
421 int32_t ofs
= STACKARG_OFS
;
423 uint32_t gprs
= REGARG_GPRS
;
424 Reg fpr
= REGARG_FIRSTFPR
;
426 MCode
*patchnfpr
= NULL
;
430 if ((ci
->flags
& CCI_CC_MASK
) != CCI_CC_CDECL
) {
431 if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_THISCALL
)
432 gprs
= (REGARG_GPRS
& 31);
433 else if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_FASTCALL
)
437 if ((void *)ci
->func
)
438 emit_call(as
, ci
->func
);
440 if ((ci
->flags
& CCI_VARARG
)) { /* Special handling for vararg calls. */
442 for (n
= 0; n
< 4 && n
< nargs
; n
++) {
443 IRIns
*ir
= IR(args
[n
]);
444 if (irt_isfp(ir
->t
)) /* Duplicate FPRs in GPRs. */
445 emit_rr(as
, XO_MOVDto
, (irt_isnum(ir
->t
) ? REX_64
: 0) | (fpr
+n
),
446 ((gprs
>> (n
*5)) & 31)); /* Either MOVD or MOVQ. */
449 patchnfpr
= --as
->mcp
; /* Indicate number of used FPRs in register al. */
450 *--as
->mcp
= XI_MOVrib
| RID_EAX
;
454 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
458 #if LJ_64 && LJ_ABI_WIN
459 /* Windows/x64 argument registers are strictly positional. */
460 r
= irt_isfp(ir
->t
) ? (fpr
<= REGARG_LASTFPR
? fpr
: 0) : (gprs
& 31);
463 /* POSIX/x64 argument registers are used in order of appearance. */
464 if (irt_isfp(ir
->t
)) {
465 r
= fpr
<= REGARG_LASTFPR
? fpr
++ : 0;
467 r
= gprs
& 31; gprs
>>= 5;
470 if (ref
&& irt_isfp(ir
->t
)) {
473 r
= gprs
& 31; gprs
>>= 5;
477 if (r
) { /* Argument is in a register. */
478 if (r
< RID_MAX_GPR
&& ref
< ASMREF_TMP1
) {
480 if (ir
->o
== IR_KINT64
)
481 emit_loadu64(as
, r
, ir_kint64(ir
)->u64
);
484 emit_loadi(as
, r
, ir
->i
);
486 lua_assert(rset_test(as
->freeset
, r
)); /* Must have been evicted. */
487 if (ra_hasreg(ir
->r
)) {
488 ra_noweak(as
, ir
->r
);
489 emit_movrr(as
, ir
, r
, ir
->r
);
491 ra_allocref(as
, ref
, RID2RSET(r
));
494 } else if (irt_isfp(ir
->t
)) { /* FP argument is on stack. */
495 lua_assert(!(irt_isfloat(ir
->t
) && irref_isk(ref
))); /* No float k. */
496 if (LJ_32
&& (ofs
& 4) && irref_isk(ref
)) {
497 /* Split stores for unaligned FP consts. */
498 emit_movmroi(as
, RID_ESP
, ofs
, (int32_t)ir_knum(ir
)->u32
.lo
);
499 emit_movmroi(as
, RID_ESP
, ofs
+4, (int32_t)ir_knum(ir
)->u32
.hi
);
501 r
= ra_alloc1(as
, ref
, RSET_FPR
);
502 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_MOVSDto
: XO_MOVSSto
,
505 ofs
+= (LJ_32
&& irt_isfloat(ir
->t
)) ? 4 : 8;
506 } else { /* Non-FP argument is on stack. */
507 if (LJ_32
&& ref
< ASMREF_TMP1
) {
508 emit_movmroi(as
, RID_ESP
, ofs
, ir
->i
);
510 r
= ra_alloc1(as
, ref
, RSET_GPR
);
511 emit_movtomro(as
, REX_64
+ r
, RID_ESP
, ofs
);
513 ofs
+= sizeof(intptr_t);
516 #if LJ_64 && !LJ_ABI_WIN
517 if (patchnfpr
) *patchnfpr
= fpr
- REGARG_FIRSTFPR
;
521 /* Setup result reg/sp for call. Evict scratch regs. */
522 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
524 RegSet drop
= RSET_SCRATCH
;
525 int hiop
= (LJ_32
&& (ir
+1)->o
== IR_HIOP
);
526 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
528 if (ra_hasreg(ir
->r
))
529 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
530 if (hiop
&& ra_hasreg((ir
+1)->r
))
531 rset_clear(drop
, (ir
+1)->r
); /* Dest reg handled below. */
532 ra_evictset(as
, drop
); /* Evictions must be performed first. */
534 if (irt_isfp(ir
->t
)) {
535 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
537 if ((ci
->flags
& CCI_CASTU64
)) {
539 if (ra_hasreg(dest
)) {
541 ra_modified(as
, dest
);
542 emit_rr(as
, XO_MOVD
, dest
|REX_64
, RID_RET
); /* Really MOVQ. */
544 if (ofs
) emit_movtomro(as
, RID_RET
|REX_64
, RID_ESP
, ofs
);
546 ra_destreg(as
, ir
, RID_FPRET
);
549 /* Number result is in x87 st0 for x86 calling convention. */
551 if (ra_hasreg(dest
)) {
553 ra_modified(as
, dest
);
554 emit_rmro(as
, irt_isnum(ir
->t
) ? XMM_MOVRM(as
) : XO_MOVSS
,
557 if ((ci
->flags
& CCI_CASTU64
)) {
558 emit_movtomro(as
, RID_RETLO
, RID_ESP
, ofs
);
559 emit_movtomro(as
, RID_RETHI
, RID_ESP
, ofs
+4);
561 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_FSTPq
: XO_FSTPd
,
562 irt_isnum(ir
->t
) ? XOg_FSTPq
: XOg_FSTPd
, RID_ESP
, ofs
);
570 lua_assert(!irt_ispri(ir
->t
));
571 ra_destreg(as
, ir
, RID_RET
);
573 } else if (LJ_32
&& irt_isfp(ir
->t
)) {
574 emit_x87op(as
, XI_FPOP
); /* Pop unused result from x87 st0. */
578 static void asm_call(ASMState
*as
, IRIns
*ir
)
580 IRRef args
[CCI_NARGS_MAX
];
581 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
582 asm_collectargs(as
, ir
, ci
, args
);
583 asm_setupresult(as
, ir
, ci
);
584 asm_gencall(as
, ci
, args
);
587 /* Return a constant function pointer or NULL for indirect calls. */
588 static void *asm_callx_func(ASMState
*as
, IRIns
*irf
, IRRef func
)
593 return (void *)irf
->i
;
595 if (irref_isk(func
)) {
597 if (irf
->o
== IR_KINT64
)
598 p
= (MCode
*)(void *)ir_k64(irf
)->u64
;
600 p
= (MCode
*)(void *)(uintptr_t)(uint32_t)irf
->i
;
601 if (p
- as
->mcp
== (int32_t)(p
- as
->mcp
))
602 return p
; /* Call target is still in +-2GB range. */
603 /* Avoid the indirect case of emit_call(). Try to hoist func addr. */
609 static void asm_callx(ASMState
*as
, IRIns
*ir
)
611 IRRef args
[CCI_NARGS_MAX
];
616 ci
.flags
= asm_callx_flags(as
, ir
);
617 asm_collectargs(as
, ir
, &ci
, args
);
618 asm_setupresult(as
, ir
, &ci
);
620 /* Have to readjust stack after non-cdecl calls due to callee cleanup. */
621 if ((ci
.flags
& CCI_CC_MASK
) != CCI_CC_CDECL
)
622 spadj
= 4 * asm_count_call_slots(as
, &ci
, args
);
624 func
= ir
->op2
; irf
= IR(func
);
625 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
626 ci
.func
= (ASMFunction
)asm_callx_func(as
, irf
, func
);
627 if (!(void *)ci
.func
) {
628 /* Use a (hoistable) non-scratch register for indirect calls. */
629 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
630 Reg r
= ra_alloc1(as
, func
, allow
);
631 if (LJ_32
) emit_spsub(as
, spadj
); /* Above code may cause restores! */
632 emit_rr(as
, XO_GROUP5
, XOg_CALL
, r
);
634 emit_spsub(as
, spadj
);
636 asm_gencall(as
, &ci
, args
);
639 /* -- Returns ------------------------------------------------------------- */
641 /* Return to lower frame. Guard that it goes to the right spot. */
642 static void asm_retf(ASMState
*as
, IRIns
*ir
)
644 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
645 void *pc
= ir_kptr(IR(ir
->op2
));
646 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
647 as
->topslot
-= (BCReg
)delta
;
648 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
649 emit_setgl(as
, base
, jit_base
);
650 emit_addptr(as
, base
, -8*delta
);
651 asm_guardcc(as
, CC_NE
);
652 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), base
, -4, ptr2addr(pc
));
655 /* -- Type conversions ---------------------------------------------------- */
657 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
659 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
660 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
661 asm_guardcc(as
, CC_P
);
662 asm_guardcc(as
, CC_NE
);
663 emit_rr(as
, XO_UCOMISD
, left
, tmp
);
664 emit_rr(as
, XO_CVTSI2SD
, tmp
, dest
);
665 if (!(as
->flags
& JIT_F_SPLIT_XMM
))
666 emit_rr(as
, XO_XORPS
, tmp
, tmp
); /* Avoid partial register stall. */
667 emit_rr(as
, XO_CVTTSD2SI
, dest
, left
);
668 /* Can't fuse since left is needed twice. */
671 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
673 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
674 Reg tmp
= ra_noreg(IR(ir
->op1
)->r
) ?
675 ra_alloc1(as
, ir
->op1
, RSET_FPR
) :
676 ra_scratch(as
, RSET_FPR
);
677 Reg right
= asm_fuseload(as
, ir
->op2
, rset_exclude(RSET_FPR
, tmp
));
678 emit_rr(as
, XO_MOVDto
, tmp
, dest
);
679 emit_mrm(as
, XO_ADDSD
, tmp
, right
);
680 ra_left(as
, tmp
, ir
->op1
);
683 static void asm_conv(ASMState
*as
, IRIns
*ir
)
685 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
686 int st64
= (st
== IRT_I64
|| st
== IRT_U64
|| (LJ_64
&& st
== IRT_P64
));
687 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
688 IRRef lref
= ir
->op1
;
689 lua_assert(irt_type(ir
->t
) != st
);
690 lua_assert(!(LJ_32
&& (irt_isint64(ir
->t
) || st64
))); /* Handled by SPLIT. */
691 if (irt_isfp(ir
->t
)) {
692 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
693 if (stfp
) { /* FP to FP conversion. */
694 Reg left
= asm_fuseload(as
, lref
, RSET_FPR
);
695 emit_mrm(as
, st
== IRT_NUM
? XO_CVTSD2SS
: XO_CVTSS2SD
, dest
, left
);
696 if (left
== dest
) return; /* Avoid the XO_XORPS. */
697 } else if (LJ_32
&& st
== IRT_U32
) { /* U32 to FP conversion on x86. */
698 /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
699 cTValue
*k
= lj_ir_k64_find(as
->J
, U64x(43380000,00000000));
700 Reg bias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
701 if (irt_isfloat(ir
->t
))
702 emit_rr(as
, XO_CVTSD2SS
, dest
, dest
);
703 emit_rr(as
, XO_SUBSD
, dest
, bias
); /* Subtract 2^52+2^51 bias. */
704 emit_rr(as
, XO_XORPS
, dest
, bias
); /* Merge bias and integer. */
705 emit_loadn(as
, bias
, k
);
706 emit_mrm(as
, XO_MOVD
, dest
, asm_fuseload(as
, lref
, RSET_GPR
));
708 } else { /* Integer to FP conversion. */
709 Reg left
= (LJ_64
&& (st
== IRT_U32
|| st
== IRT_U64
)) ?
710 ra_alloc1(as
, lref
, RSET_GPR
) :
711 asm_fuseloadm(as
, lref
, RSET_GPR
, st64
);
712 if (LJ_64
&& st
== IRT_U64
) {
713 MCLabel l_end
= emit_label(as
);
714 const void *k
= lj_ir_k64_find(as
->J
, U64x(43f00000
,00000000));
715 emit_rma(as
, XO_ADDSD
, dest
, k
); /* Add 2^64 to compensate. */
716 emit_sjcc(as
, CC_NS
, l_end
);
717 emit_rr(as
, XO_TEST
, left
|REX_64
, left
); /* Check if u64 >= 2^63. */
719 emit_mrm(as
, irt_isnum(ir
->t
) ? XO_CVTSI2SD
: XO_CVTSI2SS
,
720 dest
|((LJ_64
&& (st64
|| st
== IRT_U32
)) ? REX_64
: 0), left
);
722 if (!(as
->flags
& JIT_F_SPLIT_XMM
))
723 emit_rr(as
, XO_XORPS
, dest
, dest
); /* Avoid partial register stall. */
724 } else if (stfp
) { /* FP to integer conversion. */
725 if (irt_isguard(ir
->t
)) {
726 /* Checked conversions are only supported from number to int. */
727 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
728 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
730 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
731 x86Op op
= st
== IRT_NUM
?
732 ((ir
->op2
& IRCONV_TRUNC
) ? XO_CVTTSD2SI
: XO_CVTSD2SI
) :
733 ((ir
->op2
& IRCONV_TRUNC
) ? XO_CVTTSS2SI
: XO_CVTSS2SI
);
734 if (LJ_64
? irt_isu64(ir
->t
) : irt_isu32(ir
->t
)) {
735 /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */
736 /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */
737 Reg tmp
= ra_noreg(IR(lref
)->r
) ? ra_alloc1(as
, lref
, RSET_FPR
) :
738 ra_scratch(as
, RSET_FPR
);
739 MCLabel l_end
= emit_label(as
);
741 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, (int32_t)0x80000000);
742 emit_rr(as
, op
, dest
|REX_64
, tmp
);
744 emit_rma(as
, XO_ADDSD
, tmp
, lj_ir_k64_find(as
->J
,
745 LJ_64
? U64x(c3f00000
,00000000) : U64x(c1e00000
,00000000)));
747 emit_rma(as
, XO_ADDSS
, tmp
, lj_ir_k64_find(as
->J
,
748 LJ_64
? U64x(00000000,df800000
) : U64x(00000000,cf000000
)));
749 emit_sjcc(as
, CC_NS
, l_end
);
750 emit_rr(as
, XO_TEST
, dest
|REX_64
, dest
); /* Check if dest negative. */
751 emit_rr(as
, op
, dest
|REX_64
, tmp
);
752 ra_left(as
, tmp
, lref
);
754 Reg left
= asm_fuseload(as
, lref
, RSET_FPR
);
755 if (LJ_64
&& irt_isu32(ir
->t
))
756 emit_rr(as
, XO_MOV
, dest
, dest
); /* Zero hiword. */
759 (irt_is64(ir
->t
) || irt_isu32(ir
->t
))) ? REX_64
: 0),
763 } else if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
764 Reg left
, dest
= ra_dest(as
, ir
, RSET_GPR
);
765 RegSet allow
= RSET_GPR
;
767 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
769 op
= XO_MOVSXb
; allow
= RSET_GPR8
; dest
|= FORCE_REX
;
770 } else if (st
== IRT_U8
) {
771 op
= XO_MOVZXb
; allow
= RSET_GPR8
; dest
|= FORCE_REX
;
772 } else if (st
== IRT_I16
) {
777 left
= asm_fuseload(as
, lref
, allow
);
778 /* Add extra MOV if source is already in wrong register. */
779 if (!LJ_64
&& left
!= RID_MRM
&& !rset_test(allow
, left
)) {
780 Reg tmp
= ra_scratch(as
, allow
);
781 emit_rr(as
, op
, dest
, tmp
);
782 emit_rr(as
, XO_MOV
, tmp
, left
);
784 emit_mrm(as
, op
, dest
, left
);
786 } else { /* 32/64 bit integer conversions. */
787 if (LJ_32
) { /* Only need to handle 32/32 bit no-op (cast) on x86. */
788 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
789 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
790 } else if (irt_is64(ir
->t
)) {
791 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
792 if (st64
|| !(ir
->op2
& IRCONV_SEXT
)) {
793 /* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
794 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
795 } else { /* 32 to 64 bit sign extension. */
796 Reg left
= asm_fuseload(as
, lref
, RSET_GPR
);
797 emit_mrm(as
, XO_MOVSXd
, dest
|REX_64
, left
);
800 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
802 Reg left
= asm_fuseload(as
, lref
, RSET_GPR
);
803 /* This is either a 32 bit reg/reg mov which zeroes the hiword
804 ** or a load of the loword from a 64 bit address.
806 emit_mrm(as
, XO_MOV
, dest
, left
);
807 } else { /* 32/32 bit no-op (cast). */
808 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
814 #if LJ_32 && LJ_HASFFI
815 /* No SSE conversions to/from 64 bit on x86, so resort to ugly x87 code. */
817 /* 64 bit integer to FP conversion in 32 bit mode. */
818 static void asm_conv_fp_int64(ASMState
*as
, IRIns
*ir
)
820 Reg hi
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
821 Reg lo
= ra_alloc1(as
, (ir
-1)->op1
, rset_exclude(RSET_GPR
, hi
));
822 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
824 if (ra_hasreg(dest
)) {
826 ra_modified(as
, dest
);
827 emit_rmro(as
, irt_isnum(ir
->t
) ? XMM_MOVRM(as
) : XO_MOVSS
,
830 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_FSTPq
: XO_FSTPd
,
831 irt_isnum(ir
->t
) ? XOg_FSTPq
: XOg_FSTPd
, RID_ESP
, ofs
);
832 if (((ir
-1)->op2
& IRCONV_SRCMASK
) == IRT_U64
) {
833 /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
834 MCLabel l_end
= emit_label(as
);
835 emit_rma(as
, XO_FADDq
, XOg_FADDq
,
836 lj_ir_k64_find(as
->J
, U64x(43f00000
,00000000)));
837 emit_sjcc(as
, CC_NS
, l_end
);
838 emit_rr(as
, XO_TEST
, hi
, hi
); /* Check if u64 >= 2^63. */
840 lua_assert(((ir
-1)->op2
& IRCONV_SRCMASK
) == IRT_I64
);
842 emit_rmro(as
, XO_FILDq
, XOg_FILDq
, RID_ESP
, 0);
843 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
844 emit_rmro(as
, XO_MOVto
, hi
, RID_ESP
, 4);
845 emit_rmro(as
, XO_MOVto
, lo
, RID_ESP
, 0);
848 /* FP to 64 bit integer conversion in 32 bit mode. */
849 static void asm_conv_int64_fp(ASMState
*as
, IRIns
*ir
)
851 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
852 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
854 lua_assert(st
== IRT_NUM
|| st
== IRT_FLOAT
);
855 lua_assert(dt
== IRT_I64
|| dt
== IRT_U64
);
856 lua_assert(((ir
-1)->op2
& IRCONV_TRUNC
));
857 hi
= ra_dest(as
, ir
, RSET_GPR
);
858 lo
= ra_dest(as
, ir
-1, rset_exclude(RSET_GPR
, hi
));
859 if (ra_used(ir
-1)) emit_rmro(as
, XO_MOV
, lo
, RID_ESP
, 0);
860 /* NYI: Avoid wide-to-narrow store-to-load forwarding stall. */
861 if (!(as
->flags
& JIT_F_SSE3
)) { /* Set FPU rounding mode to default. */
862 emit_rmro(as
, XO_FLDCW
, XOg_FLDCW
, RID_ESP
, 4);
863 emit_rmro(as
, XO_MOVto
, lo
, RID_ESP
, 4);
864 emit_gri(as
, XG_ARITHi(XOg_AND
), lo
, 0xf3ff);
867 /* For inputs in [2^63,2^64-1] add -2^64 and convert again. */
868 MCLabel l_pop
, l_end
= emit_label(as
);
869 emit_x87op(as
, XI_FPOP
);
870 l_pop
= emit_label(as
);
871 emit_sjmp(as
, l_end
);
872 emit_rmro(as
, XO_MOV
, hi
, RID_ESP
, 4);
873 if ((as
->flags
& JIT_F_SSE3
))
874 emit_rmro(as
, XO_FISTTPq
, XOg_FISTTPq
, RID_ESP
, 0);
876 emit_rmro(as
, XO_FISTPq
, XOg_FISTPq
, RID_ESP
, 0);
877 emit_rma(as
, XO_FADDq
, XOg_FADDq
,
878 lj_ir_k64_find(as
->J
, U64x(c3f00000
,00000000)));
879 emit_sjcc(as
, CC_NS
, l_pop
);
880 emit_rr(as
, XO_TEST
, hi
, hi
); /* Check if out-of-range (2^63). */
882 emit_rmro(as
, XO_MOV
, hi
, RID_ESP
, 4);
883 if ((as
->flags
& JIT_F_SSE3
)) { /* Truncation is easy with SSE3. */
884 emit_rmro(as
, XO_FISTTPq
, XOg_FISTTPq
, RID_ESP
, 0);
885 } else { /* Otherwise set FPU rounding mode to truncate before the store. */
886 emit_rmro(as
, XO_FISTPq
, XOg_FISTPq
, RID_ESP
, 0);
887 emit_rmro(as
, XO_FLDCW
, XOg_FLDCW
, RID_ESP
, 0);
888 emit_rmro(as
, XO_MOVtow
, lo
, RID_ESP
, 0);
889 emit_rmro(as
, XO_ARITHw(XOg_OR
), lo
, RID_ESP
, 0);
890 emit_loadi(as
, lo
, 0xc00);
891 emit_rmro(as
, XO_FNSTCW
, XOg_FNSTCW
, RID_ESP
, 0);
894 emit_x87op(as
, XI_FDUP
);
895 emit_mrm(as
, st
== IRT_NUM
? XO_FLDq
: XO_FLDd
,
896 st
== IRT_NUM
? XOg_FLDq
: XOg_FLDd
,
897 asm_fuseload(as
, ir
->op1
, RSET_EMPTY
));
901 static void asm_strto(ASMState
*as
, IRIns
*ir
)
903 /* Force a spill slot for the destination register (if any). */
904 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
906 RegSet drop
= RSET_SCRATCH
;
907 if ((drop
& RSET_FPR
) != RSET_FPR
&& ra_hasreg(ir
->r
))
908 rset_set(drop
, ir
->r
); /* WIN64 doesn't spill all FPRs. */
909 ra_evictset(as
, drop
);
910 asm_guardcc(as
, CC_E
);
911 emit_rr(as
, XO_TEST
, RID_RET
, RID_RET
); /* Test return status. */
912 args
[0] = ir
->op1
; /* GCstr *str */
913 args
[1] = ASMREF_TMP1
; /* TValue *n */
914 asm_gencall(as
, ci
, args
);
915 /* Store the result to the spill slot or temp slots. */
916 emit_rmro(as
, XO_LEA
, ra_releasetmp(as
, ASMREF_TMP1
)|REX_64
,
917 RID_ESP
, sps_scale(ir
->s
));
920 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
922 IRIns
*irl
= IR(ir
->op1
);
926 if (irt_isnum(irl
->t
)) {
927 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
928 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
929 asm_setupresult(as
, ir
, ci
); /* GCstr * */
930 asm_gencall(as
, ci
, args
);
931 emit_rmro(as
, XO_LEA
, ra_releasetmp(as
, ASMREF_TMP1
)|REX_64
,
932 RID_ESP
, ra_spill(as
, irl
));
934 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
935 args
[1] = ir
->op1
; /* int32_t k */
936 asm_setupresult(as
, ir
, ci
); /* GCstr * */
937 asm_gencall(as
, ci
, args
);
941 /* -- Memory references --------------------------------------------------- */
943 static void asm_aref(ASMState
*as
, IRIns
*ir
)
945 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
946 asm_fusearef(as
, ir
, RSET_GPR
);
947 if (!(as
->mrm
.idx
== RID_NONE
&& as
->mrm
.ofs
== 0))
948 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
949 else if (as
->mrm
.base
!= dest
)
950 emit_rr(as
, XO_MOV
, dest
, as
->mrm
.base
);
953 /* Merge NE(HREF, niltv) check. */
954 static MCode
*merge_href_niltv(ASMState
*as
, IRIns
*ir
)
956 /* Assumes nothing else generates NE of HREF. */
957 if ((ir
[1].o
== IR_NE
|| ir
[1].o
== IR_EQ
) && ir
[1].op1
== as
->curins
&&
960 p
+= (LJ_64
&& *p
!= XI_ARITHi
) ? 7+6 : 6+6;
961 /* Ensure no loop branch inversion happened. */
962 if (p
[-6] == 0x0f && p
[-5] == XI_JCCn
+(CC_NE
^(ir
[1].o
& 1))) {
963 as
->mcp
= p
; /* Kill cmp reg, imm32 + jz exit. */
964 return p
+ *(int32_t *)(p
-4); /* Return exit address. */
970 /* Inlined hash lookup. Specialized for key type and for const keys.
971 ** The equivalent C code is:
972 ** Node *n = hashkey(t, key);
974 ** if (lj_obj_equal(&n->key, key)) return &n->val;
975 ** } while ((n = nextnode(n)));
978 static void asm_href(ASMState
*as
, IRIns
*ir
)
980 MCode
*nilexit
= merge_href_niltv(as
, ir
); /* Do this before any restores. */
981 RegSet allow
= RSET_GPR
;
982 Reg dest
= ra_dest(as
, ir
, allow
);
983 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
984 Reg key
= RID_NONE
, tmp
= RID_NONE
;
985 IRIns
*irkey
= IR(ir
->op2
);
986 int isk
= irref_isk(ir
->op2
);
987 IRType1 kt
= irkey
->t
;
989 MCLabel l_end
, l_loop
, l_next
;
992 rset_clear(allow
, tab
);
993 key
= ra_alloc1(as
, ir
->op2
, irt_isnum(kt
) ? RSET_FPR
: allow
);
995 tmp
= ra_scratch(as
, rset_exclude(allow
, key
));
998 /* Key not found in chain: jump to exit (if merged with NE) or load niltv. */
999 l_end
= emit_label(as
);
1000 if (nilexit
&& ir
[1].o
== IR_NE
) {
1001 emit_jcc(as
, CC_E
, nilexit
); /* XI_JMP is not found by lj_asm_patchexit. */
1004 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
1007 /* Follow hash chain until the end. */
1008 l_loop
= emit_sjcc_label(as
, CC_NZ
);
1009 emit_rr(as
, XO_TEST
, dest
, dest
);
1010 emit_rmro(as
, XO_MOV
, dest
, dest
, offsetof(Node
, next
));
1011 l_next
= emit_label(as
);
1013 /* Type and value comparison. */
1015 emit_jcc(as
, CC_E
, nilexit
);
1017 emit_sjcc(as
, CC_E
, l_end
);
1018 if (irt_isnum(kt
)) {
1020 /* Assumes -0.0 is already canonicalized to +0.0. */
1021 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.u32
.lo
),
1022 (int32_t)ir_knum(irkey
)->u32
.lo
);
1023 emit_sjcc(as
, CC_NE
, l_next
);
1024 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.u32
.hi
),
1025 (int32_t)ir_knum(irkey
)->u32
.hi
);
1027 emit_sjcc(as
, CC_P
, l_next
);
1028 emit_rmro(as
, XO_UCOMISD
, key
, dest
, offsetof(Node
, key
.n
));
1029 emit_sjcc(as
, CC_AE
, l_next
);
1030 /* The type check avoids NaN penalties and complaints from Valgrind. */
1032 emit_u32(as
, LJ_TISNUM
);
1033 emit_rmro(as
, XO_ARITHi
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1035 emit_i8(as
, LJ_TISNUM
);
1036 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1040 } else if (irt_islightud(kt
)) {
1041 emit_rmro(as
, XO_CMP
, key
|REX_64
, dest
, offsetof(Node
, key
.u64
));
1044 if (!irt_ispri(kt
)) {
1045 lua_assert(irt_isaddr(kt
));
1047 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.gcr
),
1048 ptr2addr(ir_kgc(irkey
)));
1050 emit_rmro(as
, XO_CMP
, key
, dest
, offsetof(Node
, key
.gcr
));
1051 emit_sjcc(as
, CC_NE
, l_next
);
1053 lua_assert(!irt_isnil(kt
));
1054 emit_i8(as
, irt_toitype(kt
));
1055 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1057 emit_sfixup(as
, l_loop
);
1060 /* Load main position relative to tab->node into dest. */
1061 khash
= isk
? ir_khash(irkey
) : 1;
1063 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, node
));
1065 emit_rmro(as
, XO_ARITH(XOg_ADD
), dest
, tab
, offsetof(GCtab
, node
));
1066 if ((as
->flags
& JIT_F_PREFER_IMUL
)) {
1067 emit_i8(as
, sizeof(Node
));
1068 emit_rr(as
, XO_IMULi8
, dest
, dest
);
1070 emit_shifti(as
, XOg_SHL
, dest
, 3);
1071 emit_rmrxo(as
, XO_LEA
, dest
, dest
, dest
, XM_SCALE2
, 0);
1074 emit_gri(as
, XG_ARITHi(XOg_AND
), dest
, (int32_t)khash
);
1075 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, hmask
));
1076 } else if (irt_isstr(kt
)) {
1077 emit_rmro(as
, XO_ARITH(XOg_AND
), dest
, key
, offsetof(GCstr
, hash
));
1078 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, hmask
));
1079 } else { /* Must match with hashrot() in lj_tab.c. */
1080 emit_rmro(as
, XO_ARITH(XOg_AND
), dest
, tab
, offsetof(GCtab
, hmask
));
1081 emit_rr(as
, XO_ARITH(XOg_SUB
), dest
, tmp
);
1082 emit_shifti(as
, XOg_ROL
, tmp
, HASH_ROT3
);
1083 emit_rr(as
, XO_ARITH(XOg_XOR
), dest
, tmp
);
1084 emit_shifti(as
, XOg_ROL
, dest
, HASH_ROT2
);
1085 emit_rr(as
, XO_ARITH(XOg_SUB
), tmp
, dest
);
1086 emit_shifti(as
, XOg_ROL
, dest
, HASH_ROT1
);
1087 emit_rr(as
, XO_ARITH(XOg_XOR
), tmp
, dest
);
1088 if (irt_isnum(kt
)) {
1089 emit_rr(as
, XO_ARITH(XOg_ADD
), dest
, dest
);
1091 emit_shifti(as
, XOg_SHR
|REX_64
, dest
, 32);
1092 emit_rr(as
, XO_MOV
, tmp
, dest
);
1093 emit_rr(as
, XO_MOVDto
, key
|REX_64
, dest
);
1095 emit_rmro(as
, XO_MOV
, dest
, RID_ESP
, ra_spill(as
, irkey
)+4);
1096 emit_rr(as
, XO_MOVDto
, key
, tmp
);
1099 emit_rr(as
, XO_MOV
, tmp
, key
);
1100 emit_rmro(as
, XO_LEA
, dest
, key
, HASH_BIAS
);
1106 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
1108 IRIns
*kslot
= IR(ir
->op2
);
1109 IRIns
*irkey
= IR(kslot
->op1
);
1110 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
1111 Reg dest
= ra_used(ir
) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
1112 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1116 lua_assert(ofs
% sizeof(Node
) == 0);
1117 if (ra_hasreg(dest
)) {
1119 if (dest
== node
&& !(as
->flags
& JIT_F_LEA_AGU
))
1120 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, ofs
);
1122 emit_rmro(as
, XO_LEA
, dest
, node
, ofs
);
1123 } else if (dest
!= node
) {
1124 emit_rr(as
, XO_MOV
, dest
, node
);
1127 asm_guardcc(as
, CC_NE
);
1129 if (!irt_ispri(irkey
->t
)) {
1130 Reg key
= ra_scratch(as
, rset_exclude(RSET_GPR
, node
));
1131 emit_rmro(as
, XO_CMP
, key
|REX_64
, node
,
1132 ofs
+ (int32_t)offsetof(Node
, key
.u64
));
1133 lua_assert(irt_isnum(irkey
->t
) || irt_isgcv(irkey
->t
));
1134 /* Assumes -0.0 is already canonicalized to +0.0. */
1135 emit_loadu64(as
, key
, irt_isnum(irkey
->t
) ? ir_knum(irkey
)->u64
:
1136 ((uint64_t)irt_toitype(irkey
->t
) << 32) |
1137 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey
)));
1139 lua_assert(!irt_isnil(irkey
->t
));
1140 emit_i8(as
, irt_toitype(irkey
->t
));
1141 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, node
,
1142 ofs
+ (int32_t)offsetof(Node
, key
.it
));
1145 l_exit
= emit_label(as
);
1146 if (irt_isnum(irkey
->t
)) {
1147 /* Assumes -0.0 is already canonicalized to +0.0. */
1148 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1149 ofs
+ (int32_t)offsetof(Node
, key
.u32
.lo
),
1150 (int32_t)ir_knum(irkey
)->u32
.lo
);
1151 emit_sjcc(as
, CC_NE
, l_exit
);
1152 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1153 ofs
+ (int32_t)offsetof(Node
, key
.u32
.hi
),
1154 (int32_t)ir_knum(irkey
)->u32
.hi
);
1156 if (!irt_ispri(irkey
->t
)) {
1157 lua_assert(irt_isgcv(irkey
->t
));
1158 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1159 ofs
+ (int32_t)offsetof(Node
, key
.gcr
),
1160 ptr2addr(ir_kgc(irkey
)));
1161 emit_sjcc(as
, CC_NE
, l_exit
);
1163 lua_assert(!irt_isnil(irkey
->t
));
1164 emit_i8(as
, irt_toitype(irkey
->t
));
1165 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, node
,
1166 ofs
+ (int32_t)offsetof(Node
, key
.it
));
1171 static void asm_newref(ASMState
*as
, IRIns
*ir
)
1173 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
1177 if (ir
->r
== RID_SINK
)
1179 args
[0] = ASMREF_L
; /* lua_State *L */
1180 args
[1] = ir
->op1
; /* GCtab *t */
1181 args
[2] = ASMREF_TMP1
; /* cTValue *key */
1182 asm_setupresult(as
, ir
, ci
); /* TValue * */
1183 asm_gencall(as
, ci
, args
);
1184 tmp
= ra_releasetmp(as
, ASMREF_TMP1
);
1185 irkey
= IR(ir
->op2
);
1186 if (irt_isnum(irkey
->t
)) {
1187 /* For numbers use the constant itself or a spill slot as a TValue. */
1188 if (irref_isk(ir
->op2
))
1189 emit_loada(as
, tmp
, ir_knum(irkey
));
1191 emit_rmro(as
, XO_LEA
, tmp
|REX_64
, RID_ESP
, ra_spill(as
, irkey
));
1193 /* Otherwise use g->tmptv to hold the TValue. */
1194 if (!irref_isk(ir
->op2
)) {
1195 Reg src
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, tmp
));
1196 emit_movtomro(as
, REX_64IR(irkey
, src
), tmp
, 0);
1197 } else if (!irt_ispri(irkey
->t
)) {
1198 emit_movmroi(as
, tmp
, 0, irkey
->i
);
1200 if (!(LJ_64
&& irt_islightud(irkey
->t
)))
1201 emit_movmroi(as
, tmp
, 4, irt_toitype(irkey
->t
));
1202 emit_loada(as
, tmp
, &J2G(as
->J
)->tmptv
);
1206 static void asm_uref(ASMState
*as
, IRIns
*ir
)
1208 /* NYI: Check that UREFO is still open and not aliasing a slot. */
1209 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1210 if (irref_isk(ir
->op1
)) {
1211 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
1212 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
1213 emit_rma(as
, XO_MOV
, dest
, v
);
1215 Reg uv
= ra_scratch(as
, RSET_GPR
);
1216 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1217 if (ir
->o
== IR_UREFC
) {
1218 emit_rmro(as
, XO_LEA
, dest
, uv
, offsetof(GCupval
, tv
));
1219 asm_guardcc(as
, CC_NE
);
1221 emit_rmro(as
, XO_ARITHib
, XOg_CMP
, uv
, offsetof(GCupval
, closed
));
1223 emit_rmro(as
, XO_MOV
, dest
, uv
, offsetof(GCupval
, v
));
1225 emit_rmro(as
, XO_MOV
, uv
, func
,
1226 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
1230 static void asm_fref(ASMState
*as
, IRIns
*ir
)
1232 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1233 asm_fusefref(as
, ir
, RSET_GPR
);
1234 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1237 static void asm_strref(ASMState
*as
, IRIns
*ir
)
1239 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1240 asm_fusestrref(as
, ir
, RSET_GPR
);
1241 if (as
->mrm
.base
== RID_NONE
)
1242 emit_loadi(as
, dest
, as
->mrm
.ofs
);
1243 else if (as
->mrm
.base
== dest
&& as
->mrm
.idx
== RID_NONE
)
1244 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, as
->mrm
.ofs
);
1246 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1249 /* -- Loads and stores ---------------------------------------------------- */
1251 static void asm_fxload(ASMState
*as
, IRIns
*ir
)
1253 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
1255 if (ir
->o
== IR_FLOAD
)
1256 asm_fusefref(as
, ir
, RSET_GPR
);
1258 asm_fusexref(as
, ir
->op1
, RSET_GPR
);
1259 /* ir->op2 is ignored -- unaligned loads are ok on x86. */
1260 switch (irt_type(ir
->t
)) {
1261 case IRT_I8
: xo
= XO_MOVSXb
; break;
1262 case IRT_U8
: xo
= XO_MOVZXb
; break;
1263 case IRT_I16
: xo
= XO_MOVSXw
; break;
1264 case IRT_U16
: xo
= XO_MOVZXw
; break;
1265 case IRT_NUM
: xo
= XMM_MOVRM(as
); break;
1266 case IRT_FLOAT
: xo
= XO_MOVSS
; break;
1268 if (LJ_64
&& irt_is64(ir
->t
))
1271 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
));
1275 emit_mrm(as
, xo
, dest
, RID_MRM
);
1278 static void asm_fxstore(ASMState
*as
, IRIns
*ir
)
1280 RegSet allow
= RSET_GPR
;
1281 Reg src
= RID_NONE
, osrc
= RID_NONE
;
1283 if (ir
->r
== RID_SINK
)
1285 /* The IRT_I16/IRT_U16 stores should never be simplified for constant
1286 ** values since mov word [mem], imm16 has a length-changing prefix.
1288 if (irt_isi16(ir
->t
) || irt_isu16(ir
->t
) || irt_isfp(ir
->t
) ||
1289 !asm_isk32(as
, ir
->op2
, &k
)) {
1290 RegSet allow8
= irt_isfp(ir
->t
) ? RSET_FPR
:
1291 (irt_isi8(ir
->t
) || irt_isu8(ir
->t
)) ? RSET_GPR8
: RSET_GPR
;
1292 src
= osrc
= ra_alloc1(as
, ir
->op2
, allow8
);
1293 if (!LJ_64
&& !rset_test(allow8
, src
)) { /* Already in wrong register. */
1294 rset_clear(allow
, osrc
);
1295 src
= ra_scratch(as
, allow8
);
1297 rset_clear(allow
, src
);
1299 if (ir
->o
== IR_FSTORE
) {
1300 asm_fusefref(as
, IR(ir
->op1
), allow
);
1302 asm_fusexref(as
, ir
->op1
, allow
);
1303 if (LJ_32
&& ir
->o
== IR_HIOP
) as
->mrm
.ofs
+= 4;
1305 if (ra_hasreg(src
)) {
1307 switch (irt_type(ir
->t
)) {
1308 case IRT_I8
: case IRT_U8
: xo
= XO_MOVtob
; src
|= FORCE_REX
; break;
1309 case IRT_I16
: case IRT_U16
: xo
= XO_MOVtow
; break;
1310 case IRT_NUM
: xo
= XO_MOVSDto
; break;
1311 case IRT_FLOAT
: xo
= XO_MOVSSto
; break;
1313 case IRT_LIGHTUD
: lua_assert(0); /* NYI: mask 64 bit lightuserdata. */
1316 if (LJ_64
&& irt_is64(ir
->t
))
1319 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
));
1323 emit_mrm(as
, xo
, src
, RID_MRM
);
1324 if (!LJ_64
&& src
!= osrc
) {
1325 ra_noweak(as
, osrc
);
1326 emit_rr(as
, XO_MOV
, src
, osrc
);
1329 if (irt_isi8(ir
->t
) || irt_isu8(ir
->t
)) {
1331 emit_mrm(as
, XO_MOVmib
, 0, RID_MRM
);
1333 lua_assert(irt_is64(ir
->t
) || irt_isint(ir
->t
) || irt_isu32(ir
->t
) ||
1336 emit_mrm(as
, XO_MOVmi
, REX_64IR(ir
, 0), RID_MRM
);
1342 static Reg
asm_load_lightud64(ASMState
*as
, IRIns
*ir
, int typecheck
)
1344 if (ra_used(ir
) || typecheck
) {
1345 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1347 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
1348 asm_guardcc(as
, CC_NE
);
1350 emit_rr(as
, XO_ARITHi8
, XOg_CMP
, tmp
);
1351 emit_shifti(as
, XOg_SAR
|REX_64
, tmp
, 47);
1352 emit_rr(as
, XO_MOV
, tmp
|REX_64
, dest
);
1361 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
1363 lua_assert(irt_isnum(ir
->t
) || irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) ||
1364 (LJ_DUALNUM
&& irt_isint(ir
->t
)));
1366 if (irt_islightud(ir
->t
)) {
1367 Reg dest
= asm_load_lightud64(as
, ir
, 1);
1368 if (ra_hasreg(dest
)) {
1369 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1370 emit_mrm(as
, XO_MOV
, dest
|REX_64
, RID_MRM
);
1376 RegSet allow
= irt_isnum(ir
->t
) ? RSET_FPR
: RSET_GPR
;
1377 Reg dest
= ra_dest(as
, ir
, allow
);
1378 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1379 emit_mrm(as
, dest
< RID_MAX_GPR
? XO_MOV
: XMM_MOVRM(as
), dest
, RID_MRM
);
1381 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1383 /* Always do the type check, even if the load result is unused. */
1385 asm_guardcc(as
, irt_isnum(ir
->t
) ? CC_AE
: CC_NE
);
1386 if (LJ_64
&& irt_type(ir
->t
) >= IRT_NUM
) {
1387 lua_assert(irt_isinteger(ir
->t
) || irt_isnum(ir
->t
));
1388 emit_u32(as
, LJ_TISNUM
);
1389 emit_mrm(as
, XO_ARITHi
, XOg_CMP
, RID_MRM
);
1391 emit_i8(as
, irt_toitype(ir
->t
));
1392 emit_mrm(as
, XO_ARITHi8
, XOg_CMP
, RID_MRM
);
1396 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
1398 if (ir
->r
== RID_SINK
)
1400 if (irt_isnum(ir
->t
)) {
1401 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
1402 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1403 emit_mrm(as
, XO_MOVSDto
, src
, RID_MRM
);
1405 } else if (irt_islightud(ir
->t
)) {
1406 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1407 asm_fuseahuref(as
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
1408 emit_mrm(as
, XO_MOVto
, src
|REX_64
, RID_MRM
);
1411 IRIns
*irr
= IR(ir
->op2
);
1412 RegSet allow
= RSET_GPR
;
1414 if (!irref_isk(ir
->op2
)) {
1415 src
= ra_alloc1(as
, ir
->op2
, allow
);
1416 rset_clear(allow
, src
);
1418 asm_fuseahuref(as
, ir
->op1
, allow
);
1419 if (ra_hasreg(src
)) {
1420 emit_mrm(as
, XO_MOVto
, src
, RID_MRM
);
1421 } else if (!irt_ispri(irr
->t
)) {
1422 lua_assert(irt_isaddr(ir
->t
) || (LJ_DUALNUM
&& irt_isinteger(ir
->t
)));
1423 emit_i32(as
, irr
->i
);
1424 emit_mrm(as
, XO_MOVmi
, 0, RID_MRM
);
1427 emit_i32(as
, (int32_t)irt_toitype(ir
->t
));
1428 emit_mrm(as
, XO_MOVmi
, 0, RID_MRM
);
1432 static void asm_sload(ASMState
*as
, IRIns
*ir
)
1434 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 4 : 0);
1437 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
1438 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
1439 lua_assert(LJ_DUALNUM
||
1440 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
1441 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
1442 Reg left
= ra_scratch(as
, RSET_FPR
);
1443 asm_tointg(as
, ir
, left
); /* Frees dest reg. Do this before base alloc. */
1444 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1445 emit_rmro(as
, XMM_MOVRM(as
), left
, base
, ofs
);
1446 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
1448 } else if (irt_islightud(t
)) {
1449 Reg dest
= asm_load_lightud64(as
, ir
, (ir
->op2
& IRSLOAD_TYPECHECK
));
1450 if (ra_hasreg(dest
)) {
1451 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1452 emit_rmro(as
, XO_MOV
, dest
|REX_64
, base
, ofs
);
1456 } else if (ra_used(ir
)) {
1457 RegSet allow
= irt_isnum(t
) ? RSET_FPR
: RSET_GPR
;
1458 Reg dest
= ra_dest(as
, ir
, allow
);
1459 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1460 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
1461 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1462 t
.irt
= irt_isint(t
) ? IRT_NUM
: IRT_INT
; /* Check for original type. */
1463 emit_rmro(as
, irt_isint(t
) ? XO_CVTSI2SD
: XO_CVTSD2SI
, dest
, base
, ofs
);
1464 } else if (irt_isnum(t
)) {
1465 emit_rmro(as
, XMM_MOVRM(as
), dest
, base
, ofs
);
1467 emit_rmro(as
, XO_MOV
, dest
, base
, ofs
);
1470 if (!(ir
->op2
& IRSLOAD_TYPECHECK
))
1471 return; /* No type check: avoid base alloc. */
1472 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1474 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1475 /* Need type check, even if the load result is unused. */
1476 asm_guardcc(as
, irt_isnum(t
) ? CC_AE
: CC_NE
);
1477 if (LJ_64
&& irt_type(t
) >= IRT_NUM
) {
1478 lua_assert(irt_isinteger(t
) || irt_isnum(t
));
1479 emit_u32(as
, LJ_TISNUM
);
1480 emit_rmro(as
, XO_ARITHi
, XOg_CMP
, base
, ofs
+4);
1482 emit_i8(as
, irt_toitype(t
));
1483 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, base
, ofs
+4);
1488 /* -- Allocations --------------------------------------------------------- */
1491 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1493 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1494 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1495 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1496 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1497 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1499 lua_assert(sz
!= CTSIZE_INVALID
);
1501 args
[0] = ASMREF_L
; /* lua_State *L */
1502 args
[1] = ASMREF_TMP1
; /* MSize size */
1504 asm_setupresult(as
, ir
, ci
); /* GCcdata * */
1506 /* Initialize immutable cdata object. */
1507 if (ir
->o
== IR_CNEWI
) {
1508 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1510 Reg r64
= sz
== 8 ? REX_64
: 0;
1511 if (irref_isk(ir
->op2
)) {
1512 IRIns
*irk
= IR(ir
->op2
);
1513 uint64_t k
= irk
->o
== IR_KINT64
? ir_k64(irk
)->u64
:
1514 (uint64_t)(uint32_t)irk
->i
;
1515 if (sz
== 4 || checki32((int64_t)k
)) {
1516 emit_i32(as
, (int32_t)k
);
1517 emit_rmro(as
, XO_MOVmi
, r64
, RID_RET
, sizeof(GCcdata
));
1519 emit_movtomro(as
, RID_ECX
+ r64
, RID_RET
, sizeof(GCcdata
));
1520 emit_loadu64(as
, RID_ECX
, k
);
1523 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1524 emit_movtomro(as
, r
+ r64
, RID_RET
, sizeof(GCcdata
));
1527 int32_t ofs
= sizeof(GCcdata
);
1530 lua_assert(ir
->o
== IR_HIOP
);
1533 if (irref_isk(ir
->op2
)) {
1534 emit_movmroi(as
, RID_RET
, ofs
, IR(ir
->op2
)->i
);
1536 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1537 emit_movtomro(as
, r
, RID_RET
, ofs
);
1538 rset_clear(allow
, r
);
1540 if (ofs
== sizeof(GCcdata
)) break;
1544 lua_assert(sz
== 4 || sz
== 8);
1547 /* Combine initialization of marked, gct and ctypeid. */
1548 emit_movtomro(as
, RID_ECX
, RID_RET
, offsetof(GCcdata
, marked
));
1549 emit_gri(as
, XG_ARITHi(XOg_OR
), RID_ECX
,
1550 (int32_t)((~LJ_TCDATA
<<8)+(ctypeid
<<16)));
1551 emit_gri(as
, XG_ARITHi(XOg_AND
), RID_ECX
, LJ_GC_WHITES
);
1552 emit_opgl(as
, XO_MOVZXb
, RID_ECX
, gc
.currentwhite
);
1554 asm_gencall(as
, ci
, args
);
1555 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP1
), (int32_t)(sz
+sizeof(GCcdata
)));
1558 #define asm_cnew(as, ir) ((void)0)
1561 /* -- Write barriers ------------------------------------------------------ */
1563 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1565 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1566 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1567 MCLabel l_end
= emit_label(as
);
1568 emit_movtomro(as
, tmp
, tab
, offsetof(GCtab
, gclist
));
1569 emit_setgl(as
, tab
, gc
.grayagain
);
1570 emit_getgl(as
, tmp
, gc
.grayagain
);
1571 emit_i8(as
, ~LJ_GC_BLACK
);
1572 emit_rmro(as
, XO_ARITHib
, XOg_AND
, tab
, offsetof(GCtab
, marked
));
1573 emit_sjcc(as
, CC_Z
, l_end
);
1574 emit_i8(as
, LJ_GC_BLACK
);
1575 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, tab
, offsetof(GCtab
, marked
));
1578 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1580 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1584 /* No need for other object barriers (yet). */
1585 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1586 ra_evictset(as
, RSET_SCRATCH
);
1587 l_end
= emit_label(as
);
1588 args
[0] = ASMREF_TMP1
; /* global_State *g */
1589 args
[1] = ir
->op1
; /* TValue *tv */
1590 asm_gencall(as
, ci
, args
);
1591 emit_loada(as
, ra_releasetmp(as
, ASMREF_TMP1
), J2G(as
->J
));
1592 obj
= IR(ir
->op1
)->r
;
1593 emit_sjcc(as
, CC_Z
, l_end
);
1594 emit_i8(as
, LJ_GC_WHITES
);
1595 if (irref_isk(ir
->op2
)) {
1596 GCobj
*vp
= ir_kgc(IR(ir
->op2
));
1597 emit_rma(as
, XO_GROUP3b
, XOg_TEST
, &vp
->gch
.marked
);
1599 Reg val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_SCRATCH
&RSET_GPR
, obj
));
1600 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, val
, (int32_t)offsetof(GChead
, marked
));
1602 emit_sjcc(as
, CC_Z
, l_end
);
1603 emit_i8(as
, LJ_GC_BLACK
);
1604 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, obj
,
1605 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1608 /* -- FP/int arithmetic and logic operations ------------------------------ */
1610 /* Load reference onto x87 stack. Force a spill to memory if needed. */
1611 static void asm_x87load(ASMState
*as
, IRRef ref
)
1613 IRIns
*ir
= IR(ref
);
1614 if (ir
->o
== IR_KNUM
) {
1615 cTValue
*tv
= ir_knum(ir
);
1616 if (tvispzero(tv
)) /* Use fldz only for +0. */
1617 emit_x87op(as
, XI_FLDZ
);
1618 else if (tvispone(tv
))
1619 emit_x87op(as
, XI_FLD1
);
1621 emit_rma(as
, XO_FLDq
, XOg_FLDq
, tv
);
1622 } else if (ir
->o
== IR_CONV
&& ir
->op2
== IRCONV_NUM_INT
&& !ra_used(ir
) &&
1623 !irref_isk(ir
->op1
) && mayfuse(as
, ir
->op1
)) {
1624 IRIns
*iri
= IR(ir
->op1
);
1625 emit_rmro(as
, XO_FILDd
, XOg_FILDd
, RID_ESP
, ra_spill(as
, iri
));
1627 emit_mrm(as
, XO_FLDq
, XOg_FLDq
, asm_fuseload(as
, ref
, RSET_EMPTY
));
1631 /* Try to rejoin pow from EXP2, MUL and LOG2 (if still unsplit). */
1632 static int fpmjoin_pow(ASMState
*as
, IRIns
*ir
)
1634 IRIns
*irp
= IR(ir
->op1
);
1635 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1636 IRIns
*irpp
= IR(irp
->op1
);
1637 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1638 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1639 /* The modified regs must match with the *.dasc implementation. */
1640 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM2
+1)|RID2RSET(RID_EAX
);
1642 if (ra_hasreg(ir
->r
))
1643 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1644 ra_evictset(as
, drop
);
1645 ra_destreg(as
, ir
, RID_XMM0
);
1646 emit_call(as
, lj_vm_pow_sse
);
1647 irx
= IR(irpp
->op1
);
1648 if (ra_noreg(irx
->r
) && ra_gethint(irx
->r
) == RID_XMM1
)
1649 irx
->r
= RID_INIT
; /* Avoid allocating xmm1 for x. */
1650 ra_left(as
, RID_XMM0
, irpp
->op1
);
1651 ra_left(as
, RID_XMM1
, irp
->op2
);
1658 static void asm_fpmath(ASMState
*as
, IRIns
*ir
)
1660 IRFPMathOp fpm
= ir
->o
== IR_FPMATH
? (IRFPMathOp
)ir
->op2
: IRFPM_OTHER
;
1661 if (fpm
== IRFPM_SQRT
) {
1662 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1663 Reg left
= asm_fuseload(as
, ir
->op1
, RSET_FPR
);
1664 emit_mrm(as
, XO_SQRTSD
, dest
, left
);
1665 } else if (fpm
<= IRFPM_TRUNC
) {
1666 if (as
->flags
& JIT_F_SSE4_1
) { /* SSE4.1 has a rounding instruction. */
1667 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1668 Reg left
= asm_fuseload(as
, ir
->op1
, RSET_FPR
);
1669 /* ROUNDSD has a 4-byte opcode which doesn't fit in x86Op.
1670 ** Let's pretend it's a 3-byte opcode, and compensate afterwards.
1671 ** This is atrocious, but the alternatives are much worse.
1673 /* Round down/up/trunc == 1001/1010/1011. */
1674 emit_i8(as
, 0x09 + fpm
);
1675 emit_mrm(as
, XO_ROUNDSD
, dest
, left
);
1676 if (LJ_64
&& as
->mcp
[1] != (MCode
)(XO_ROUNDSD
>> 16)) {
1677 as
->mcp
[0] = as
->mcp
[1]; as
->mcp
[1] = 0x0f; /* Swap 0F and REX. */
1679 *--as
->mcp
= 0x66; /* 1st byte of ROUNDSD opcode. */
1680 } else { /* Call helper functions for SSE2 variant. */
1681 /* The modified regs must match with the *.dasc implementation. */
1682 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM3
+1)|RID2RSET(RID_EAX
);
1683 if (ra_hasreg(ir
->r
))
1684 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1685 ra_evictset(as
, drop
);
1686 ra_destreg(as
, ir
, RID_XMM0
);
1687 emit_call(as
, fpm
== IRFPM_FLOOR
? lj_vm_floor_sse
:
1688 fpm
== IRFPM_CEIL
? lj_vm_ceil_sse
: lj_vm_trunc_sse
);
1689 ra_left(as
, RID_XMM0
, ir
->op1
);
1691 } else if (fpm
== IRFPM_EXP2
&& fpmjoin_pow(as
, ir
)) {
1692 /* Rejoined to pow(). */
1693 } else { /* Handle x87 ops. */
1694 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
1696 if (ra_hasreg(dest
)) {
1698 ra_modified(as
, dest
);
1699 emit_rmro(as
, XMM_MOVRM(as
), dest
, RID_ESP
, ofs
);
1701 emit_rmro(as
, XO_FSTPq
, XOg_FSTPq
, RID_ESP
, ofs
);
1702 switch (fpm
) { /* st0 = lj_vm_*(st0) */
1703 case IRFPM_EXP
: emit_call(as
, lj_vm_exp_x87
); break;
1704 case IRFPM_EXP2
: emit_call(as
, lj_vm_exp2_x87
); break;
1705 case IRFPM_SIN
: emit_x87op(as
, XI_FSIN
); break;
1706 case IRFPM_COS
: emit_x87op(as
, XI_FCOS
); break;
1707 case IRFPM_TAN
: emit_x87op(as
, XI_FPOP
); emit_x87op(as
, XI_FPTAN
); break;
1708 case IRFPM_LOG
: case IRFPM_LOG2
: case IRFPM_LOG10
:
1709 /* Note: the use of fyl2xp1 would be pointless here. When computing
1710 ** log(1.0+eps) the precision is already lost after 1.0 is added.
1711 ** Subtracting 1.0 won't recover it. OTOH math.log1p would make sense.
1713 emit_x87op(as
, XI_FYL2X
); break;
1717 emit_x87op(as
, XI_FPATAN
); asm_x87load(as
, ir
->op2
); break;
1719 emit_x87op(as
, XI_FPOP1
); emit_x87op(as
, XI_FSCALE
); break;
1720 default: lua_assert(0); break;
1723 default: lua_assert(0); break;
1725 asm_x87load(as
, ir
->op1
);
1727 case IRFPM_LOG
: emit_x87op(as
, XI_FLDLN2
); break;
1728 case IRFPM_LOG2
: emit_x87op(as
, XI_FLD1
); break;
1729 case IRFPM_LOG10
: emit_x87op(as
, XI_FLDLG2
); break;
1731 if (ir
->o
== IR_LDEXP
) asm_x87load(as
, ir
->op2
);
1738 static void asm_fppowi(ASMState
*as
, IRIns
*ir
)
1740 /* The modified regs must match with the *.dasc implementation. */
1741 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM1
+1)|RID2RSET(RID_EAX
);
1742 if (ra_hasreg(ir
->r
))
1743 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1744 ra_evictset(as
, drop
);
1745 ra_destreg(as
, ir
, RID_XMM0
);
1746 emit_call(as
, lj_vm_powi_sse
);
1747 ra_left(as
, RID_XMM0
, ir
->op1
);
1748 ra_left(as
, RID_EAX
, ir
->op2
);
1751 #if LJ_64 && LJ_HASFFI
1752 static void asm_arith64(ASMState
*as
, IRIns
*ir
, IRCallID id
)
1754 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
1758 asm_setupresult(as
, ir
, ci
);
1759 asm_gencall(as
, ci
, args
);
1763 static void asm_intmod(ASMState
*as
, IRIns
*ir
)
1765 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_vm_modi
];
1769 asm_setupresult(as
, ir
, ci
);
1770 asm_gencall(as
, ci
, args
);
1773 static int asm_swapops(ASMState
*as
, IRIns
*ir
)
1775 IRIns
*irl
= IR(ir
->op1
);
1776 IRIns
*irr
= IR(ir
->op2
);
1777 lua_assert(ra_noreg(irr
->r
));
1778 if (!irm_iscomm(lj_ir_mode
[ir
->o
]))
1779 return 0; /* Can't swap non-commutative operations. */
1780 if (irref_isk(ir
->op2
))
1781 return 0; /* Don't swap constants to the left. */
1782 if (ra_hasreg(irl
->r
))
1783 return 1; /* Swap if left already has a register. */
1784 if (ra_samehint(ir
->r
, irr
->r
))
1785 return 1; /* Swap if dest and right have matching hints. */
1786 if (as
->curins
> as
->loopref
) { /* In variant part? */
1787 if (ir
->op2
< as
->loopref
&& !irt_isphi(irr
->t
))
1788 return 0; /* Keep invariants on the right. */
1789 if (ir
->op1
< as
->loopref
&& !irt_isphi(irl
->t
))
1790 return 1; /* Swap invariants to the right. */
1792 if (opisfusableload(irl
->o
))
1793 return 1; /* Swap fusable loads to the right. */
1794 return 0; /* Otherwise don't swap. */
1797 static void asm_fparith(ASMState
*as
, IRIns
*ir
, x86Op xo
)
1799 IRRef lref
= ir
->op1
;
1800 IRRef rref
= ir
->op2
;
1801 RegSet allow
= RSET_FPR
;
1803 Reg right
= IR(rref
)->r
;
1804 if (ra_hasreg(right
)) {
1805 rset_clear(allow
, right
);
1806 ra_noweak(as
, right
);
1808 dest
= ra_dest(as
, ir
, allow
);
1811 } else if (ra_noreg(right
)) {
1812 if (asm_swapops(as
, ir
)) {
1813 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1815 right
= asm_fuseload(as
, rref
, rset_clear(allow
, dest
));
1817 emit_mrm(as
, xo
, dest
, right
);
1818 ra_left(as
, dest
, lref
);
1821 static void asm_intarith(ASMState
*as
, IRIns
*ir
, x86Arith xa
)
1823 IRRef lref
= ir
->op1
;
1824 IRRef rref
= ir
->op2
;
1825 RegSet allow
= RSET_GPR
;
1828 if (as
->flagmcp
== as
->mcp
) { /* Drop test r,r instruction. */
1830 as
->mcp
+= (LJ_64
&& *as
->mcp
< XI_TESTb
) ? 3 : 2;
1832 right
= IR(rref
)->r
;
1833 if (ra_hasreg(right
)) {
1834 rset_clear(allow
, right
);
1835 ra_noweak(as
, right
);
1837 dest
= ra_dest(as
, ir
, allow
);
1840 } else if (ra_noreg(right
) && !asm_isk32(as
, rref
, &k
)) {
1841 if (asm_swapops(as
, ir
)) {
1842 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1844 right
= asm_fuseloadm(as
, rref
, rset_clear(allow
, dest
), irt_is64(ir
->t
));
1846 if (irt_isguard(ir
->t
)) /* For IR_ADDOV etc. */
1847 asm_guardcc(as
, CC_O
);
1848 if (xa
!= XOg_X_IMUL
) {
1849 if (ra_hasreg(right
))
1850 emit_mrm(as
, XO_ARITH(xa
), REX_64IR(ir
, dest
), right
);
1852 emit_gri(as
, XG_ARITHi(xa
), REX_64IR(ir
, dest
), k
);
1853 } else if (ra_hasreg(right
)) { /* IMUL r, mrm. */
1854 emit_mrm(as
, XO_IMUL
, REX_64IR(ir
, dest
), right
);
1855 } else { /* IMUL r, r, k. */
1856 /* NYI: use lea/shl/add/sub (FOLD only does 2^k) depending on CPU. */
1857 Reg left
= asm_fuseloadm(as
, lref
, RSET_GPR
, irt_is64(ir
->t
));
1859 if (checki8(k
)) { emit_i8(as
, k
); xo
= XO_IMULi8
;
1860 } else { emit_i32(as
, k
); xo
= XO_IMULi
; }
1861 emit_mrm(as
, xo
, REX_64IR(ir
, dest
), left
);
1864 ra_left(as
, dest
, lref
);
1867 /* LEA is really a 4-operand ADD with an independent destination register,
1868 ** up to two source registers and an immediate. One register can be scaled
1869 ** by 1, 2, 4 or 8. This can be used to avoid moves or to fuse several
1872 ** Currently only a few common cases are supported:
1873 ** - 3-operand ADD: y = a+b; y = a+k with a and b already allocated
1874 ** - Left ADD fusion: y = (a+b)+k; y = (a+k)+b
1875 ** - Right ADD fusion: y = a+(b+k)
1876 ** The ommited variants have already been reduced by FOLD.
1878 ** There are more fusion opportunities, like gathering shifts or joining
1879 ** common references. But these are probably not worth the trouble, since
1880 ** array indexing is not decomposed and already makes use of all fields
1881 ** of the ModRM operand.
1883 static int asm_lea(ASMState
*as
, IRIns
*ir
)
1885 IRIns
*irl
= IR(ir
->op1
);
1886 IRIns
*irr
= IR(ir
->op2
);
1887 RegSet allow
= RSET_GPR
;
1889 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
1890 as
->mrm
.scale
= XM_SCALE1
;
1892 if (ra_hasreg(irl
->r
)) {
1893 rset_clear(allow
, irl
->r
);
1894 ra_noweak(as
, irl
->r
);
1895 as
->mrm
.base
= irl
->r
;
1896 if (irref_isk(ir
->op2
) || ra_hasreg(irr
->r
)) {
1897 /* The PHI renaming logic does a better job in some cases. */
1898 if (ra_hasreg(ir
->r
) &&
1899 ((irt_isphi(irl
->t
) && as
->phireg
[ir
->r
] == ir
->op1
) ||
1900 (irt_isphi(irr
->t
) && as
->phireg
[ir
->r
] == ir
->op2
)))
1902 if (irref_isk(ir
->op2
)) {
1903 as
->mrm
.ofs
= irr
->i
;
1905 rset_clear(allow
, irr
->r
);
1906 ra_noweak(as
, irr
->r
);
1907 as
->mrm
.idx
= irr
->r
;
1909 } else if (irr
->o
== IR_ADD
&& mayfuse(as
, ir
->op2
) &&
1910 irref_isk(irr
->op2
)) {
1911 Reg idx
= ra_alloc1(as
, irr
->op1
, allow
);
1912 rset_clear(allow
, idx
);
1913 as
->mrm
.idx
= (uint8_t)idx
;
1914 as
->mrm
.ofs
= IR(irr
->op2
)->i
;
1918 } else if (ir
->op1
!= ir
->op2
&& irl
->o
== IR_ADD
&& mayfuse(as
, ir
->op1
) &&
1919 (irref_isk(ir
->op2
) || irref_isk(irl
->op2
))) {
1920 Reg idx
, base
= ra_alloc1(as
, irl
->op1
, allow
);
1921 rset_clear(allow
, base
);
1922 as
->mrm
.base
= (uint8_t)base
;
1923 if (irref_isk(ir
->op2
)) {
1924 as
->mrm
.ofs
= irr
->i
;
1925 idx
= ra_alloc1(as
, irl
->op2
, allow
);
1927 as
->mrm
.ofs
= IR(irl
->op2
)->i
;
1928 idx
= ra_alloc1(as
, ir
->op2
, allow
);
1930 rset_clear(allow
, idx
);
1931 as
->mrm
.idx
= (uint8_t)idx
;
1935 dest
= ra_dest(as
, ir
, allow
);
1936 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1937 return 1; /* Success. */
1940 static void asm_add(ASMState
*as
, IRIns
*ir
)
1942 if (irt_isnum(ir
->t
))
1943 asm_fparith(as
, ir
, XO_ADDSD
);
1944 else if ((as
->flags
& JIT_F_LEA_AGU
) || as
->flagmcp
== as
->mcp
||
1945 irt_is64(ir
->t
) || !asm_lea(as
, ir
))
1946 asm_intarith(as
, ir
, XOg_ADD
);
1949 static void asm_neg_not(ASMState
*as
, IRIns
*ir
, x86Group3 xg
)
1951 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1952 emit_rr(as
, XO_GROUP3
, REX_64IR(ir
, xg
), dest
);
1953 ra_left(as
, dest
, ir
->op1
);
1956 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int cc
)
1958 Reg right
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1959 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1960 if (irref_isk(rref
)) { lref
= rref
; rref
= ir
->op1
; }
1961 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, dest
));
1962 emit_rr(as
, XO_CMOV
+ (cc
<<24), REX_64IR(ir
, dest
), right
);
1963 emit_rr(as
, XO_CMP
, REX_64IR(ir
, dest
), right
);
1964 ra_left(as
, dest
, lref
);
1967 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1969 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1970 as
->mcp
= emit_op(XO_BSWAP
+ ((dest
&7) << 24),
1971 REX_64IR(ir
, 0), dest
, 0, as
->mcp
, 1);
1972 ra_left(as
, dest
, ir
->op1
);
1975 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, x86Shift xs
)
1977 IRRef rref
= ir
->op2
;
1978 IRIns
*irr
= IR(rref
);
1980 if (irref_isk(rref
)) { /* Constant shifts. */
1982 dest
= ra_dest(as
, ir
, RSET_GPR
);
1983 shift
= irr
->i
& (irt_is64(ir
->t
) ? 63 : 31);
1986 case 1: emit_rr(as
, XO_SHIFT1
, REX_64IR(ir
, xs
), dest
); break;
1987 default: emit_shifti(as
, REX_64IR(ir
, xs
), dest
, shift
); break;
1989 } else { /* Variable shifts implicitly use register cl (i.e. ecx). */
1991 dest
= ra_dest(as
, ir
, rset_exclude(RSET_GPR
, RID_ECX
));
1992 if (dest
== RID_ECX
) {
1993 dest
= ra_scratch(as
, rset_exclude(RSET_GPR
, RID_ECX
));
1994 emit_rr(as
, XO_MOV
, RID_ECX
, dest
);
1997 if (ra_noreg(right
))
1998 right
= ra_allocref(as
, rref
, RID2RSET(RID_ECX
));
1999 else if (right
!= RID_ECX
)
2000 ra_scratch(as
, RID2RSET(RID_ECX
));
2001 emit_rr(as
, XO_SHIFTcl
, REX_64IR(ir
, xs
), dest
);
2002 ra_noweak(as
, right
);
2003 if (right
!= RID_ECX
)
2004 emit_rr(as
, XO_MOV
, RID_ECX
, right
);
2006 ra_left(as
, dest
, ir
->op1
);
2008 ** Note: avoid using the flags resulting from a shift or rotate!
2009 ** All of them cause a partial flag stall, except for r,1 shifts
2010 ** (but not rotates). And a shift count of 0 leaves the flags unmodified.
2014 /* -- Comparisons --------------------------------------------------------- */
2016 /* Virtual flags for unordered FP comparisons. */
2017 #define VCC_U 0x1000 /* Unordered. */
2018 #define VCC_P 0x2000 /* Needs extra CC_P branch. */
2019 #define VCC_S 0x4000 /* Swap avoids CC_P branch. */
2020 #define VCC_PS (VCC_P|VCC_S)
2022 /* Map of comparisons to flags. ORDER IR. */
2023 #define COMPFLAGS(ci, cin, cu, cf) ((ci)+((cu)<<4)+((cin)<<8)+(cf))
2024 static const uint16_t asm_compmap
[IR_ABC
+1] = {
2025 /* signed non-eq unsigned flags */
2026 /* LT */ COMPFLAGS(CC_GE
, CC_G
, CC_AE
, VCC_PS
),
2027 /* GE */ COMPFLAGS(CC_L
, CC_L
, CC_B
, 0),
2028 /* LE */ COMPFLAGS(CC_G
, CC_G
, CC_A
, VCC_PS
),
2029 /* GT */ COMPFLAGS(CC_LE
, CC_L
, CC_BE
, 0),
2030 /* ULT */ COMPFLAGS(CC_AE
, CC_A
, CC_AE
, VCC_U
),
2031 /* UGE */ COMPFLAGS(CC_B
, CC_B
, CC_B
, VCC_U
|VCC_PS
),
2032 /* ULE */ COMPFLAGS(CC_A
, CC_A
, CC_A
, VCC_U
),
2033 /* UGT */ COMPFLAGS(CC_BE
, CC_B
, CC_BE
, VCC_U
|VCC_PS
),
2034 /* EQ */ COMPFLAGS(CC_NE
, CC_NE
, CC_NE
, VCC_P
),
2035 /* NE */ COMPFLAGS(CC_E
, CC_E
, CC_E
, VCC_U
|VCC_P
),
2036 /* ABC */ COMPFLAGS(CC_BE
, CC_B
, CC_BE
, VCC_U
|VCC_PS
) /* Same as UGT. */
2039 /* FP and integer comparisons. */
2040 static void asm_comp(ASMState
*as
, IRIns
*ir
, uint32_t cc
)
2042 if (irt_isnum(ir
->t
)) {
2043 IRRef lref
= ir
->op1
;
2044 IRRef rref
= ir
->op2
;
2048 ** An extra CC_P branch is required to preserve ordered/unordered
2049 ** semantics for FP comparisons. This can be avoided by swapping
2050 ** the operands and inverting the condition (except for EQ and UNE).
2051 ** So always try to swap if possible.
2053 ** Another option would be to swap operands to achieve better memory
2054 ** operand fusion. But it's unlikely that this outweighs the cost
2055 ** of the extra branches.
2057 if (cc
& VCC_S
) { /* Swap? */
2058 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
2059 cc
^= (VCC_PS
|(5<<4)); /* A <-> B, AE <-> BE, PS <-> none */
2061 left
= ra_alloc1(as
, lref
, RSET_FPR
);
2062 right
= asm_fuseload(as
, rref
, rset_exclude(RSET_FPR
, left
));
2063 l_around
= emit_label(as
);
2064 asm_guardcc(as
, cc
>> 4);
2065 if (cc
& VCC_P
) { /* Extra CC_P branch required? */
2066 if (!(cc
& VCC_U
)) {
2067 asm_guardcc(as
, CC_P
); /* Branch to exit for ordered comparisons. */
2068 } else if (l_around
!= as
->invmcp
) {
2069 emit_sjcc(as
, CC_P
, l_around
); /* Branch around for unordered. */
2071 /* Patched to mcloop by asm_loop_fixup. */
2074 emit_sjcc(as
, CC_P
, as
->mcp
);
2076 emit_jcc(as
, CC_P
, as
->mcp
);
2079 emit_mrm(as
, XO_UCOMISD
, left
, right
);
2081 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
2082 IROp leftop
= (IROp
)(IR(lref
)->o
);
2083 Reg r64
= REX_64IR(ir
, 0);
2085 lua_assert(irt_is64(ir
->t
) || irt_isint(ir
->t
) ||
2086 irt_isu32(ir
->t
) || irt_isaddr(ir
->t
) || irt_isu8(ir
->t
));
2087 /* Swap constants (only for ABC) and fusable loads to the right. */
2088 if (irref_isk(lref
) || (!irref_isk(rref
) && opisfusableload(leftop
))) {
2089 if ((cc
& 0xc) == 0xc) cc
^= 0x53; /* L <-> G, LE <-> GE */
2090 else if ((cc
& 0xa) == 0x2) cc
^= 0x55; /* A <-> B, AE <-> BE */
2091 lref
= ir
->op2
; rref
= ir
->op1
;
2093 if (asm_isk32(as
, rref
, &imm
)) {
2094 IRIns
*irl
= IR(lref
);
2095 /* Check wether we can use test ins. Not for unsigned, since CF=0. */
2096 int usetest
= (imm
== 0 && (cc
& 0xa) != 0x2);
2097 if (usetest
&& irl
->o
== IR_BAND
&& irl
+1 == ir
&& !ra_used(irl
)) {
2098 /* Combine comp(BAND(ref, r/imm), 0) into test mrm, r/imm. */
2099 Reg right
, left
= RID_NONE
;
2100 RegSet allow
= RSET_GPR
;
2101 if (!asm_isk32(as
, irl
->op2
, &imm
)) {
2102 left
= ra_alloc1(as
, irl
->op2
, allow
);
2103 rset_clear(allow
, left
);
2104 } else { /* Try to Fuse IRT_I8/IRT_U8 loads, too. See below. */
2105 IRIns
*irll
= IR(irl
->op1
);
2106 if (opisfusableload((IROp
)irll
->o
) &&
2107 (irt_isi8(irll
->t
) || irt_isu8(irll
->t
))) {
2108 IRType1 origt
= irll
->t
; /* Temporarily flip types. */
2109 irll
->t
.irt
= (irll
->t
.irt
& ~IRT_TYPE
) | IRT_INT
;
2110 as
->curins
--; /* Skip to BAND to avoid failing in noconflict(). */
2111 right
= asm_fuseload(as
, irl
->op1
, RSET_GPR
);
2114 if (right
!= RID_MRM
) goto test_nofuse
;
2115 /* Fusion succeeded, emit test byte mrm, imm8. */
2116 asm_guardcc(as
, cc
);
2117 emit_i8(as
, (imm
& 0xff));
2118 emit_mrm(as
, XO_GROUP3b
, XOg_TEST
, RID_MRM
);
2122 as
->curins
--; /* Skip to BAND to avoid failing in noconflict(). */
2123 right
= asm_fuseloadm(as
, irl
->op1
, allow
, r64
);
2124 as
->curins
++; /* Undo the above. */
2126 asm_guardcc(as
, cc
);
2127 if (ra_noreg(left
)) {
2129 emit_mrm(as
, XO_GROUP3
, r64
+ XOg_TEST
, right
);
2131 emit_mrm(as
, XO_TEST
, r64
+ left
, right
);
2135 if (opisfusableload((IROp
)irl
->o
) &&
2136 ((irt_isu8(irl
->t
) && checku8(imm
)) ||
2137 ((irt_isi8(irl
->t
) || irt_isi16(irl
->t
)) && checki8(imm
)) ||
2138 (irt_isu16(irl
->t
) && checku16(imm
) && checki8((int16_t)imm
)))) {
2139 /* Only the IRT_INT case is fused by asm_fuseload.
2140 ** The IRT_I8/IRT_U8 loads and some IRT_I16/IRT_U16 loads
2141 ** are handled here.
2142 ** Note that cmp word [mem], imm16 should not be generated,
2143 ** since it has a length-changing prefix. Compares of a word
2144 ** against a sign-extended imm8 are ok, however.
2146 IRType1 origt
= irl
->t
; /* Temporarily flip types. */
2147 irl
->t
.irt
= (irl
->t
.irt
& ~IRT_TYPE
) | IRT_INT
;
2148 left
= asm_fuseload(as
, lref
, RSET_GPR
);
2150 if (left
== RID_MRM
) { /* Fusion succeeded? */
2151 if (irt_isu8(irl
->t
) || irt_isu16(irl
->t
))
2152 cc
>>= 4; /* Need unsigned compare. */
2153 asm_guardcc(as
, cc
);
2155 emit_mrm(as
, (irt_isi8(origt
) || irt_isu8(origt
)) ?
2156 XO_ARITHib
: XO_ARITHiw8
, r64
+ XOg_CMP
, RID_MRM
);
2158 } /* Otherwise handle register case as usual. */
2160 left
= asm_fuseloadm(as
, lref
,
2161 irt_isu8(ir
->t
) ? RSET_GPR8
: RSET_GPR
, r64
);
2163 asm_guardcc(as
, cc
);
2164 if (usetest
&& left
!= RID_MRM
) {
2165 /* Use test r,r instead of cmp r,0. */
2167 if (irt_isu8(ir
->t
)) {
2168 lua_assert(ir
->o
== IR_EQ
|| ir
->o
== IR_NE
);
2170 if (!rset_test(RSET_RANGE(RID_EAX
, RID_EBX
+1), left
)) {
2175 emit_mrm(as
, XO_GROUP3
, XOg_TEST
, left
);
2180 emit_rr(as
, xo
, r64
+ left
, left
);
2181 if (irl
+1 == ir
) /* Referencing previous ins? */
2182 as
->flagmcp
= as
->mcp
; /* Set flag to drop test r,r if possible. */
2184 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), r64
+ left
, imm
);
2188 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
2189 Reg right
= asm_fuseloadm(as
, rref
, rset_exclude(RSET_GPR
, left
), r64
);
2190 asm_guardcc(as
, cc
);
2191 emit_mrm(as
, XO_CMP
, r64
+ left
, right
);
2196 #if LJ_32 && LJ_HASFFI
2197 /* 64 bit integer comparisons in 32 bit mode. */
2198 static void asm_comp_int64(ASMState
*as
, IRIns
*ir
)
2200 uint32_t cc
= asm_compmap
[(ir
-1)->o
];
2201 RegSet allow
= RSET_GPR
;
2202 Reg lefthi
= RID_NONE
, leftlo
= RID_NONE
;
2203 Reg righthi
= RID_NONE
, rightlo
= RID_NONE
;
2207 as
->curins
--; /* Skip loword ins. Avoids failing in noconflict(), too. */
2209 /* Allocate/fuse hiword operands. */
2210 if (irref_isk(ir
->op2
)) {
2211 lefthi
= asm_fuseload(as
, ir
->op1
, allow
);
2213 lefthi
= ra_alloc1(as
, ir
->op1
, allow
);
2214 righthi
= asm_fuseload(as
, ir
->op2
, allow
);
2215 if (righthi
== RID_MRM
) {
2216 if (as
->mrm
.base
!= RID_NONE
) rset_clear(allow
, as
->mrm
.base
);
2217 if (as
->mrm
.idx
!= RID_NONE
) rset_clear(allow
, as
->mrm
.idx
);
2219 rset_clear(allow
, righthi
);
2222 mrm
= as
->mrm
; /* Save state for hiword instruction. */
2224 /* Allocate/fuse loword operands. */
2225 if (irref_isk((ir
-1)->op2
)) {
2226 leftlo
= asm_fuseload(as
, (ir
-1)->op1
, allow
);
2228 leftlo
= ra_alloc1(as
, (ir
-1)->op1
, allow
);
2229 rightlo
= asm_fuseload(as
, (ir
-1)->op2
, allow
);
2230 if (rightlo
== RID_MRM
) {
2231 if (as
->mrm
.base
!= RID_NONE
) rset_clear(allow
, as
->mrm
.base
);
2232 if (as
->mrm
.idx
!= RID_NONE
) rset_clear(allow
, as
->mrm
.idx
);
2234 rset_clear(allow
, rightlo
);
2238 /* All register allocations must be performed _before_ this point. */
2239 l_around
= emit_label(as
);
2240 as
->invmcp
= as
->flagmcp
= NULL
; /* Cannot use these optimizations. */
2242 /* Loword comparison and branch. */
2243 asm_guardcc(as
, cc
>> 4); /* Always use unsigned compare for loword. */
2244 if (ra_noreg(rightlo
)) {
2245 int32_t imm
= IR((ir
-1)->op2
)->i
;
2246 if (imm
== 0 && ((cc
>> 4) & 0xa) != 0x2 && leftlo
!= RID_MRM
)
2247 emit_rr(as
, XO_TEST
, leftlo
, leftlo
);
2249 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), leftlo
, imm
);
2251 emit_mrm(as
, XO_CMP
, leftlo
, rightlo
);
2254 /* Hiword comparison and branches. */
2255 if ((cc
& 15) != CC_NE
)
2256 emit_sjcc(as
, CC_NE
, l_around
); /* Hiword unequal: skip loword compare. */
2257 if ((cc
& 15) != CC_E
)
2258 asm_guardcc(as
, cc
>> 8); /* Hiword compare without equality check. */
2259 as
->mrm
= mrm
; /* Restore state. */
2260 if (ra_noreg(righthi
)) {
2261 int32_t imm
= IR(ir
->op2
)->i
;
2262 if (imm
== 0 && (cc
& 0xa) != 0x2 && lefthi
!= RID_MRM
)
2263 emit_rr(as
, XO_TEST
, lefthi
, lefthi
);
2265 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), lefthi
, imm
);
2267 emit_mrm(as
, XO_CMP
, lefthi
, righthi
);
2272 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
2274 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
2275 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
2277 #if LJ_32 && LJ_HASFFI
2278 /* HIOP is marked as a store because it needs its own DCE logic. */
2279 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
2280 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
2281 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
2282 if (usehi
|| uselo
) {
2283 if (irt_isfp(ir
->t
))
2284 asm_conv_fp_int64(as
, ir
);
2286 asm_conv_int64_fp(as
, ir
);
2288 as
->curins
--; /* Always skip the CONV. */
2290 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
2291 asm_comp_int64(as
, ir
);
2293 } else if ((ir
-1)->o
== IR_XSTORE
) {
2294 if ((ir
-1)->r
!= RID_SINK
)
2295 asm_fxstore(as
, ir
);
2298 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
2299 switch ((ir
-1)->o
) {
2303 asm_intarith(as
, ir
, XOg_ADC
);
2304 asm_intarith(as
, ir
-1, XOg_ADD
);
2309 asm_intarith(as
, ir
, XOg_SBB
);
2310 asm_intarith(as
, ir
-1, XOg_SUB
);
2313 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2314 emit_rr(as
, XO_GROUP3
, XOg_NEG
, dest
);
2316 emit_rr(as
, XO_ARITHi8
, XOg_ADC
, dest
);
2317 ra_left(as
, dest
, ir
->op1
);
2319 asm_neg_not(as
, ir
-1, XOg_NEG
);
2325 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark lo op as used. */
2328 /* Nothing to do here. Handled by CNEWI itself. */
2330 default: lua_assert(0); break;
2333 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused on x64 or without FFI. */
2337 /* -- Stack handling ------------------------------------------------------ */
2339 /* Check Lua stack size for overflow. Use exit handler as fallback. */
2340 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
2341 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
2343 /* Try to get an unused temp. register, otherwise spill/restore eax. */
2344 Reg pbase
= irp
? irp
->r
: RID_BASE
;
2345 Reg r
= allow
? rset_pickbot(allow
) : RID_EAX
;
2346 emit_jcc(as
, CC_B
, exitstub_addr(as
->J
, exitno
));
2347 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
2348 emit_rmro(as
, XO_MOV
, r
|REX_64
, RID_ESP
, 0);
2351 emit_gri(as
, XG_ARITHi(XOg_CMP
), r
, (int32_t)(8*topslot
));
2352 if (ra_hasreg(pbase
) && pbase
!= r
)
2353 emit_rr(as
, XO_ARITH(XOg_SUB
), r
, pbase
);
2355 emit_rmro(as
, XO_ARITH(XOg_SUB
), r
, RID_NONE
,
2356 ptr2addr(&J2G(as
->J
)->jit_base
));
2357 emit_rmro(as
, XO_MOV
, r
, r
, offsetof(lua_State
, maxstack
));
2358 emit_getgl(as
, r
, jit_L
);
2359 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
2360 emit_rmro(as
, XO_MOVto
, r
|REX_64
, RID_ESP
, 0);
2363 /* Restore Lua stack from on-trace state. */
2364 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
2366 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
2367 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
2368 MSize n
, nent
= snap
->nent
;
2369 /* Store the value of all modified slots to the Lua stack. */
2370 for (n
= 0; n
< nent
; n
++) {
2371 SnapEntry sn
= map
[n
];
2372 BCReg s
= snap_slot(sn
);
2373 int32_t ofs
= 8*((int32_t)s
-1);
2374 IRRef ref
= snap_ref(sn
);
2375 IRIns
*ir
= IR(ref
);
2376 if ((sn
& SNAP_NORESTORE
))
2378 if (irt_isnum(ir
->t
)) {
2379 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
2380 emit_rmro(as
, XO_MOVSDto
, src
, RID_BASE
, ofs
);
2382 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) ||
2383 (LJ_DUALNUM
&& irt_isinteger(ir
->t
)));
2384 if (!irref_isk(ref
)) {
2385 Reg src
= ra_alloc1(as
, ref
, rset_exclude(RSET_GPR
, RID_BASE
));
2386 emit_movtomro(as
, REX_64IR(ir
, src
), RID_BASE
, ofs
);
2387 } else if (!irt_ispri(ir
->t
)) {
2388 emit_movmroi(as
, RID_BASE
, ofs
, ir
->i
);
2390 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
2391 if (s
!= 0) /* Do not overwrite link to previous frame. */
2392 emit_movmroi(as
, RID_BASE
, ofs
+4, (int32_t)(*flinks
--));
2394 if (!(LJ_64
&& irt_islightud(ir
->t
)))
2395 emit_movmroi(as
, RID_BASE
, ofs
+4, irt_toitype(ir
->t
));
2400 lua_assert(map
+ nent
== flinks
);
2403 /* -- GC handling --------------------------------------------------------- */
2405 /* Check GC threshold and do one or more GC steps. */
2406 static void asm_gc_check(ASMState
*as
)
2408 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
2412 ra_evictset(as
, RSET_SCRATCH
);
2413 l_end
= emit_label(as
);
2414 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2415 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
2416 emit_rr(as
, XO_TEST
, RID_RET
, RID_RET
);
2417 args
[0] = ASMREF_TMP1
; /* global_State *g */
2418 args
[1] = ASMREF_TMP2
; /* MSize steps */
2419 asm_gencall(as
, ci
, args
);
2420 tmp
= ra_releasetmp(as
, ASMREF_TMP1
);
2421 emit_loada(as
, tmp
, J2G(as
->J
));
2422 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP2
), as
->gcsteps
);
2423 /* Jump around GC step if GC total < GC threshold. */
2424 emit_sjcc(as
, CC_B
, l_end
);
2425 emit_opgl(as
, XO_ARITH(XOg_CMP
), tmp
, gc
.threshold
);
2426 emit_getgl(as
, tmp
, gc
.total
);
2431 /* -- Loop handling ------------------------------------------------------- */
2433 /* Fixup the loop branch. */
2434 static void asm_loop_fixup(ASMState
*as
)
2436 MCode
*p
= as
->mctop
;
2437 MCode
*target
= as
->mcp
;
2438 if (as
->realign
) { /* Realigned loops use short jumps. */
2439 as
->realign
= NULL
; /* Stop another retry. */
2440 lua_assert(((intptr_t)target
& 15) == 0);
2441 if (as
->loopinv
) { /* Inverted loop branch? */
2444 lua_assert(target
- p
>= -128);
2445 p
[-1] = (MCode
)(target
- p
); /* Patch sjcc. */
2446 if (as
->loopinv
== 2)
2447 p
[-3] = (MCode
)(target
- p
+ 2); /* Patch opt. short jp. */
2449 lua_assert(target
- p
>= -128);
2450 p
[-1] = (MCode
)(int8_t)(target
- p
); /* Patch short jmp. */
2456 if (as
->loopinv
) { /* Inverted loop branch? */
2457 /* asm_guardcc already inverted the jcc and patched the jmp. */
2460 *(int32_t *)(p
-4) = (int32_t)(target
- p
); /* Patch jcc. */
2461 if (as
->loopinv
== 2) {
2462 *(int32_t *)(p
-10) = (int32_t)(target
- p
+ 6); /* Patch opt. jp. */
2465 } else { /* Otherwise just patch jmp. */
2466 *(int32_t *)(p
-4) = (int32_t)(target
- p
);
2469 /* Realign small loops and shorten the loop branch. */
2470 if (newloop
>= p
- 128) {
2471 as
->realign
= newloop
; /* Force a retry and remember alignment. */
2472 as
->curins
= as
->stopins
; /* Abort asm_trace now. */
2473 as
->T
->nins
= as
->orignins
; /* Remove any added renames. */
2478 /* -- Head of trace ------------------------------------------------------- */
2480 /* Coalesce BASE register for a root trace. */
2481 static void asm_head_root_base(ASMState
*as
)
2483 IRIns
*ir
= IR(REF_BASE
);
2487 if (rset_test(as
->modset
, r
))
2488 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2490 emit_rr(as
, XO_MOV
, r
, RID_BASE
);
2494 /* Coalesce or reload BASE register for a side trace. */
2495 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
2497 IRIns
*ir
= IR(REF_BASE
);
2501 if (rset_test(as
->modset
, r
))
2502 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2504 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
2505 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
2506 rset_clear(allow
, irp
->r
);
2507 emit_rr(as
, XO_MOV
, r
, irp
->r
); /* Move from coalesced parent reg. */
2509 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
2515 /* -- Tail of trace ------------------------------------------------------- */
2517 /* Fixup the tail code. */
2518 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
2520 /* Note: don't use as->mcp swap + emit_*: emit_op overwrites more bytes. */
2521 MCode
*p
= as
->mctop
;
2523 int32_t spadj
= as
->T
->spadjust
;
2525 p
-= ((as
->flags
& JIT_F_LEA_AGU
) ? 7 : 6) + (LJ_64
? 1 : 0);
2528 /* Patch stack adjustment. */
2529 if (checki8(spadj
)) {
2535 *(int32_t *)p1
= spadj
;
2537 if ((as
->flags
& JIT_F_LEA_AGU
)) {
2541 p1
[-3] = (MCode
)XI_LEA
;
2542 p1
[-2] = MODRM(checki8(spadj
) ? XM_OFS8
: XM_OFS32
, RID_ESP
, RID_ESP
);
2543 p1
[-1] = MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
2548 p1
[-2] = (MCode
)(checki8(spadj
) ? XI_ARITHi8
: XI_ARITHi
);
2549 p1
[-1] = MODRM(XM_REG
, XOg_ADD
, RID_ESP
);
2552 /* Patch exit branch. */
2553 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
2554 *(int32_t *)(p
-4) = jmprel(p
, target
);
2556 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
2557 for (q
= as
->mctop
-1; q
>= p
; q
--)
2562 /* Prepare tail of code. */
2563 static void asm_tail_prep(ASMState
*as
)
2565 MCode
*p
= as
->mctop
;
2566 /* Realign and leave room for backwards loop branch or exit branch. */
2568 int i
= ((int)(intptr_t)as
->realign
) & 15;
2569 /* Fill unused mcode tail with NOPs to make the prefetcher happy. */
2573 p
-= (as
->loopinv
? 5 : 2); /* Space for short/near jmp. */
2575 p
-= 5; /* Space for exit branch (near jmp). */
2578 as
->invmcp
= as
->mcp
= p
;
2580 /* Leave room for ESP adjustment: add esp, imm or lea esp, [esp+imm] */
2581 as
->mcp
= p
- (((as
->flags
& JIT_F_LEA_AGU
) ? 7 : 6) + (LJ_64
? 1 : 0));
2586 /* -- Instruction dispatch ------------------------------------------------ */
2588 /* Assemble a single instruction. */
2589 static void asm_ir(ASMState
*as
, IRIns
*ir
)
2591 switch ((IROp
)ir
->o
) {
2592 /* Miscellaneous ops. */
2593 case IR_LOOP
: asm_loop(as
); break;
2594 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
2596 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
2597 case IR_PHI
: asm_phi(as
, ir
); break;
2598 case IR_HIOP
: asm_hiop(as
, ir
); break;
2599 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
2601 /* Guarded assertions. */
2602 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
2603 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
2604 case IR_EQ
: case IR_NE
: case IR_ABC
:
2605 asm_comp(as
, ir
, asm_compmap
[ir
->o
]);
2608 case IR_RETF
: asm_retf(as
, ir
); break;
2611 case IR_BNOT
: asm_neg_not(as
, ir
, XOg_NOT
); break;
2612 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
2614 case IR_BAND
: asm_intarith(as
, ir
, XOg_AND
); break;
2615 case IR_BOR
: asm_intarith(as
, ir
, XOg_OR
); break;
2616 case IR_BXOR
: asm_intarith(as
, ir
, XOg_XOR
); break;
2618 case IR_BSHL
: asm_bitshift(as
, ir
, XOg_SHL
); break;
2619 case IR_BSHR
: asm_bitshift(as
, ir
, XOg_SHR
); break;
2620 case IR_BSAR
: asm_bitshift(as
, ir
, XOg_SAR
); break;
2621 case IR_BROL
: asm_bitshift(as
, ir
, XOg_ROL
); break;
2622 case IR_BROR
: asm_bitshift(as
, ir
, XOg_ROR
); break;
2624 /* Arithmetic ops. */
2625 case IR_ADD
: asm_add(as
, ir
); break;
2627 if (irt_isnum(ir
->t
))
2628 asm_fparith(as
, ir
, XO_SUBSD
);
2629 else /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
2630 asm_intarith(as
, ir
, XOg_SUB
);
2633 if (irt_isnum(ir
->t
))
2634 asm_fparith(as
, ir
, XO_MULSD
);
2636 asm_intarith(as
, ir
, XOg_X_IMUL
);
2639 #if LJ_64 && LJ_HASFFI
2640 if (!irt_isnum(ir
->t
))
2641 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_divi64
:
2642 IRCALL_lj_carith_divu64
);
2645 asm_fparith(as
, ir
, XO_DIVSD
);
2648 #if LJ_64 && LJ_HASFFI
2649 if (!irt_isint(ir
->t
))
2650 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_modi64
:
2651 IRCALL_lj_carith_modu64
);
2658 if (irt_isnum(ir
->t
))
2659 asm_fparith(as
, ir
, XO_XORPS
);
2661 asm_neg_not(as
, ir
, XOg_NEG
);
2663 case IR_ABS
: asm_fparith(as
, ir
, XO_ANDPS
); break;
2666 if (irt_isnum(ir
->t
))
2667 asm_fparith(as
, ir
, XO_MINSD
);
2669 asm_min_max(as
, ir
, CC_G
);
2672 if (irt_isnum(ir
->t
))
2673 asm_fparith(as
, ir
, XO_MAXSD
);
2675 asm_min_max(as
, ir
, CC_L
);
2678 case IR_FPMATH
: case IR_ATAN2
: case IR_LDEXP
:
2682 #if LJ_64 && LJ_HASFFI
2683 if (!irt_isnum(ir
->t
))
2684 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_powi64
:
2685 IRCALL_lj_carith_powu64
);
2691 /* Overflow-checking arithmetic ops. Note: don't use LEA here! */
2692 case IR_ADDOV
: asm_intarith(as
, ir
, XOg_ADD
); break;
2693 case IR_SUBOV
: asm_intarith(as
, ir
, XOg_SUB
); break;
2694 case IR_MULOV
: asm_intarith(as
, ir
, XOg_X_IMUL
); break;
2696 /* Memory references. */
2697 case IR_AREF
: asm_aref(as
, ir
); break;
2698 case IR_HREF
: asm_href(as
, ir
); break;
2699 case IR_HREFK
: asm_hrefk(as
, ir
); break;
2700 case IR_NEWREF
: asm_newref(as
, ir
); break;
2701 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
2702 case IR_FREF
: asm_fref(as
, ir
); break;
2703 case IR_STRREF
: asm_strref(as
, ir
); break;
2705 /* Loads and stores. */
2706 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
2707 asm_ahuvload(as
, ir
);
2709 case IR_FLOAD
: case IR_XLOAD
: asm_fxload(as
, ir
); break;
2710 case IR_SLOAD
: asm_sload(as
, ir
); break;
2712 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
2713 case IR_FSTORE
: case IR_XSTORE
: asm_fxstore(as
, ir
); break;
2716 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
2717 case IR_TNEW
: asm_tnew(as
, ir
); break;
2718 case IR_TDUP
: asm_tdup(as
, ir
); break;
2719 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
2721 /* Write barriers. */
2722 case IR_TBAR
: asm_tbar(as
, ir
); break;
2723 case IR_OBAR
: asm_obar(as
, ir
); break;
2725 /* Type conversions. */
2726 case IR_TOBIT
: asm_tobit(as
, ir
); break;
2727 case IR_CONV
: asm_conv(as
, ir
); break;
2728 case IR_TOSTR
: asm_tostr(as
, ir
); break;
2729 case IR_STRTO
: asm_strto(as
, ir
); break;
2732 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
2733 case IR_CALLXS
: asm_callx(as
, ir
); break;
2734 case IR_CARG
: break;
2737 setintV(&as
->J
->errinfo
, ir
->o
);
2738 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
2743 /* -- Trace setup --------------------------------------------------------- */
2745 /* Ensure there are enough stack slots for call arguments. */
2746 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
2748 IRRef args
[CCI_NARGS_MAX
];
2750 asm_collectargs(as
, ir
, ci
, args
);
2751 nslots
= asm_count_call_slots(as
, ci
, args
);
2752 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
2753 as
->evenspill
= nslots
;
2755 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
2757 return irt_isfp(ir
->t
) ? REGSP_INIT
: REGSP_HINT(RID_RET
);
2761 /* Target-specific setup. */
2762 static void asm_setup_target(ASMState
*as
)
2764 asm_exitstub_setup(as
, as
->T
->nsnap
);
2767 /* -- Trace patching ------------------------------------------------------ */
2769 /* Patch exit jumps of existing machine code to a new target. */
2770 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
2772 MCode
*p
= T
->mcode
;
2773 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
2774 MSize len
= T
->szmcode
;
2775 MCode
*px
= exitstub_addr(J
, exitno
) - 6;
2776 MCode
*pe
= p
+len
-6;
2777 uint32_t stateaddr
= u32ptr(&J2G(J
)->vmstate
);
2778 if (len
> 5 && p
[len
-5] == XI_JMP
&& p
+len
-6 + *(int32_t *)(p
+len
-4) == px
)
2779 *(int32_t *)(p
+len
-4) = jmprel(p
+len
, target
);
2780 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
2782 if (*(uint32_t *)(p
+(LJ_64
? 3 : 2)) == stateaddr
&& p
[0] == XI_MOVmi
) {
2783 p
+= LJ_64
? 11 : 10;
2787 for (; p
< pe
; p
++) {
2788 if ((*(uint16_t *)p
& 0xf0ff) == 0x800f && p
+ *(int32_t *)(p
+2) == px
) {
2789 *(int32_t *)(p
+2) = jmprel(p
+6, target
);
2793 lj_mcode_sync(T
->mcode
, T
->mcode
+ T
->szmcode
);
2794 lj_mcode_patch(J
, mcarea
, 1);