2 ** x86/x64 IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Guard handling ------------------------------------------------------ */
8 /* Generate an exit stub group at the bottom of the reserved MCode memory. */
9 static MCode
*asm_exitstub_gen(ASMState
*as
, ExitNo group
)
11 ExitNo i
, groupofs
= (group
*EXITSTUBS_PER_GROUP
) & 0xff;
12 MCode
*mxp
= as
->mcbot
;
13 MCode
*mxpstart
= mxp
;
14 if (mxp
+ (2+2)*EXITSTUBS_PER_GROUP
+8+5 >= as
->mctop
)
16 /* Push low byte of exitno for each exit stub. */
17 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)groupofs
;
18 for (i
= 1; i
< EXITSTUBS_PER_GROUP
; i
++) {
19 *mxp
++ = XI_JMPs
; *mxp
++ = (MCode
)((2+2)*(EXITSTUBS_PER_GROUP
- i
) - 2);
20 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)(groupofs
+ i
);
22 /* Push the high byte of the exitno for each exit stub group. */
23 *mxp
++ = XI_PUSHi8
; *mxp
++ = (MCode
)((group
*EXITSTUBS_PER_GROUP
)>>8);
24 /* Store DISPATCH at original stack slot 0. Account for the two push ops. */
26 *mxp
++ = MODRM(XM_OFS8
, 0, RID_ESP
);
27 *mxp
++ = MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
28 *mxp
++ = 2*sizeof(void *);
29 *(int32_t *)mxp
= ptr2addr(J2GG(as
->J
)->dispatch
); mxp
+= 4;
30 /* Jump to exit handler which fills in the ExitState. */
31 *mxp
++ = XI_JMP
; mxp
+= 4;
32 *((int32_t *)(mxp
-4)) = jmprel(mxp
, (MCode
*)(void *)lj_vm_exit_handler
);
33 /* Commit the code for this group (even if assembly fails later on). */
34 lj_mcode_commitbot(as
->J
, mxp
);
36 as
->mclim
= as
->mcbot
+ MCLIM_REDZONE
;
40 /* Setup all needed exit stubs. */
41 static void asm_exitstub_setup(ASMState
*as
, ExitNo nexits
)
44 if (nexits
>= EXITSTUBS_PER_GROUP
*LJ_MAX_EXITSTUBGR
)
45 lj_trace_err(as
->J
, LJ_TRERR_SNAPOV
);
46 for (i
= 0; i
< (nexits
+EXITSTUBS_PER_GROUP
-1)/EXITSTUBS_PER_GROUP
; i
++)
47 if (as
->J
->exitstubgroup
[i
] == NULL
)
48 as
->J
->exitstubgroup
[i
] = asm_exitstub_gen(as
, i
);
51 /* Emit conditional branch to exit for guard.
52 ** It's important to emit this *after* all registers have been allocated,
53 ** because rematerializations may invalidate the flags.
55 static void asm_guardcc(ASMState
*as
, int cc
)
57 MCode
*target
= exitstub_addr(as
->J
, as
->snapno
);
59 if (LJ_UNLIKELY(p
== as
->invmcp
)) {
61 *(int32_t *)(p
+1) = jmprel(p
+5, target
);
65 emit_sjcc(as
, cc
, target
);
69 emit_jcc(as
, cc
, target
);
72 /* -- Memory operand fusion ----------------------------------------------- */
74 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
75 #define CONFLICT_SEARCH_LIM 31
77 /* Check if a reference is a signed 32 bit constant. */
78 static int asm_isk32(ASMState
*as
, IRRef ref
, int32_t *k
)
82 if (ir
->o
!= IR_KINT64
) {
85 } else if (checki32((int64_t)ir_kint64(ir
)->u64
)) {
86 *k
= (int32_t)ir_kint64(ir
)->u64
;
93 /* Check if there's no conflicting instruction between curins and ref.
94 ** Also avoid fusing loads if there are multiple references.
96 static int noconflict(ASMState
*as
, IRRef ref
, IROp conflict
, int noload
)
100 if (i
> ref
+ CONFLICT_SEARCH_LIM
)
101 return 0; /* Give up, ref is too far away. */
103 if (ir
[i
].o
== conflict
)
104 return 0; /* Conflict found. */
105 else if (!noload
&& (ir
[i
].op1
== ref
|| ir
[i
].op2
== ref
))
108 return 1; /* Ok, no conflict. */
111 /* Fuse array base into memory operand. */
112 static IRRef
asm_fuseabase(ASMState
*as
, IRRef ref
)
114 IRIns
*irb
= IR(ref
);
116 if (irb
->o
== IR_FLOAD
) {
117 IRIns
*ira
= IR(irb
->op1
);
118 lua_assert(irb
->op2
== IRFL_TAB_ARRAY
);
119 /* We can avoid the FLOAD of t->array for colocated arrays. */
120 if (ira
->o
== IR_TNEW
&& ira
->op1
<= LJ_MAX_COLOSIZE
&&
121 !neverfuse(as
) && noconflict(as
, irb
->op1
, IR_NEWREF
, 1)) {
122 as
->mrm
.ofs
= (int32_t)sizeof(GCtab
); /* Ofs to colocated array. */
123 return irb
->op1
; /* Table obj. */
125 } else if (irb
->o
== IR_ADD
&& irref_isk(irb
->op2
)) {
126 /* Fuse base offset (vararg load). */
127 as
->mrm
.ofs
= IR(irb
->op2
)->i
;
130 return ref
; /* Otherwise use the given array base. */
133 /* Fuse array reference into memory operand. */
134 static void asm_fusearef(ASMState
*as
, IRIns
*ir
, RegSet allow
)
137 lua_assert(ir
->o
== IR_AREF
);
138 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, asm_fuseabase(as
, ir
->op1
), allow
);
140 if (irref_isk(ir
->op2
)) {
141 as
->mrm
.ofs
+= 8*irx
->i
;
142 as
->mrm
.idx
= RID_NONE
;
144 rset_clear(allow
, as
->mrm
.base
);
145 as
->mrm
.scale
= XM_SCALE8
;
146 /* Fuse a constant ADD (e.g. t[i+1]) into the offset.
147 ** Doesn't help much without ABCelim, but reduces register pressure.
149 if (!LJ_64
&& /* Has bad effects with negative index on x64. */
150 mayfuse(as
, ir
->op2
) && ra_noreg(irx
->r
) &&
151 irx
->o
== IR_ADD
&& irref_isk(irx
->op2
)) {
152 as
->mrm
.ofs
+= 8*IR(irx
->op2
)->i
;
153 as
->mrm
.idx
= (uint8_t)ra_alloc1(as
, irx
->op1
, allow
);
155 as
->mrm
.idx
= (uint8_t)ra_alloc1(as
, ir
->op2
, allow
);
160 /* Fuse array/hash/upvalue reference into memory operand.
161 ** Caveat: this may allocate GPRs for the base/idx registers. Be sure to
162 ** pass the final allow mask, excluding any GPRs used for other inputs.
163 ** In particular: 2-operand GPR instructions need to call ra_dest() first!
165 static void asm_fuseahuref(ASMState
*as
, IRRef ref
, RegSet allow
)
168 if (ra_noreg(ir
->r
)) {
169 switch ((IROp
)ir
->o
) {
171 if (mayfuse(as
, ref
)) {
172 asm_fusearef(as
, ir
, allow
);
177 if (mayfuse(as
, ref
)) {
178 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ir
->op1
, allow
);
179 as
->mrm
.ofs
= (int32_t)(IR(ir
->op2
)->op2
* sizeof(Node
));
180 as
->mrm
.idx
= RID_NONE
;
185 if (irref_isk(ir
->op1
)) {
186 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
187 GCupval
*uv
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
;
188 as
->mrm
.ofs
= ptr2addr(&uv
->tv
);
189 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
194 lua_assert(ir
->o
== IR_HREF
|| ir
->o
== IR_NEWREF
|| ir
->o
== IR_UREFO
||
199 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ref
, allow
);
201 as
->mrm
.idx
= RID_NONE
;
204 /* Fuse FLOAD/FREF reference into memory operand. */
205 static void asm_fusefref(ASMState
*as
, IRIns
*ir
, RegSet allow
)
207 lua_assert(ir
->o
== IR_FLOAD
|| ir
->o
== IR_FREF
);
208 as
->mrm
.ofs
= field_ofs
[ir
->op2
];
209 as
->mrm
.idx
= RID_NONE
;
210 if (irref_isk(ir
->op1
)) {
211 as
->mrm
.ofs
+= IR(ir
->op1
)->i
;
212 as
->mrm
.base
= RID_NONE
;
214 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ir
->op1
, allow
);
218 /* Fuse string reference into memory operand. */
219 static void asm_fusestrref(ASMState
*as
, IRIns
*ir
, RegSet allow
)
222 lua_assert(ir
->o
== IR_STRREF
);
223 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
224 as
->mrm
.scale
= XM_SCALE1
;
225 as
->mrm
.ofs
= sizeof(GCstr
);
226 if (irref_isk(ir
->op1
)) {
227 as
->mrm
.ofs
+= IR(ir
->op1
)->i
;
229 Reg r
= ra_alloc1(as
, ir
->op1
, allow
);
230 rset_clear(allow
, r
);
231 as
->mrm
.base
= (uint8_t)r
;
234 if (irref_isk(ir
->op2
)) {
235 as
->mrm
.ofs
+= irr
->i
;
238 /* Fuse a constant add into the offset, e.g. string.sub(s, i+10). */
239 if (!LJ_64
&& /* Has bad effects with negative index on x64. */
240 mayfuse(as
, ir
->op2
) && irr
->o
== IR_ADD
&& irref_isk(irr
->op2
)) {
241 as
->mrm
.ofs
+= IR(irr
->op2
)->i
;
242 r
= ra_alloc1(as
, irr
->op1
, allow
);
244 r
= ra_alloc1(as
, ir
->op2
, allow
);
246 if (as
->mrm
.base
== RID_NONE
)
247 as
->mrm
.base
= (uint8_t)r
;
249 as
->mrm
.idx
= (uint8_t)r
;
253 static void asm_fusexref(ASMState
*as
, IRRef ref
, RegSet allow
)
256 as
->mrm
.idx
= RID_NONE
;
257 if (ir
->o
== IR_KPTR
|| ir
->o
== IR_KKPTR
) {
259 as
->mrm
.base
= RID_NONE
;
260 } else if (ir
->o
== IR_STRREF
) {
261 asm_fusestrref(as
, ir
, allow
);
264 if (canfuse(as
, ir
) && ir
->o
== IR_ADD
&& ra_noreg(ir
->r
)) {
265 /* Gather (base+idx*sz)+ofs as emitted by cdata ptr/array indexing. */
269 if (asm_isk32(as
, ir
->op2
, &as
->mrm
.ofs
)) { /* Recognize x+ofs. */
272 if (!(ir
->o
== IR_ADD
&& canfuse(as
, ir
) && ra_noreg(ir
->r
)))
275 as
->mrm
.scale
= XM_SCALE1
;
279 if (!(irx
->o
== IR_BSHL
|| irx
->o
== IR_ADD
)) { /* Try other operand. */
284 if (canfuse(as
, irx
) && ra_noreg(irx
->r
)) {
285 if (irx
->o
== IR_BSHL
&& irref_isk(irx
->op2
) && IR(irx
->op2
)->i
<= 3) {
286 /* Recognize idx<<b with b = 0-3, corresponding to sz = (1),2,4,8. */
288 as
->mrm
.scale
= (uint8_t)(IR(irx
->op2
)->i
<< 6);
289 } else if (irx
->o
== IR_ADD
&& irx
->op1
== irx
->op2
) {
290 /* FOLD does idx*2 ==> idx<<1 ==> idx+idx. */
292 as
->mrm
.scale
= XM_SCALE2
;
295 r
= ra_alloc1(as
, idx
, allow
);
296 rset_clear(allow
, r
);
297 as
->mrm
.idx
= (uint8_t)r
;
300 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, ref
, allow
);
304 /* Fuse load into memory operand. */
305 static Reg
asm_fuseload(ASMState
*as
, IRRef ref
, RegSet allow
)
308 if (ra_hasreg(ir
->r
)) {
309 if (allow
!= RSET_EMPTY
) { /* Fast path. */
310 ra_noweak(as
, ir
->r
);
314 /* Force a spill if only memory operands are allowed (asm_x87load). */
315 as
->mrm
.base
= RID_ESP
;
316 as
->mrm
.ofs
= ra_spill(as
, ir
);
317 as
->mrm
.idx
= RID_NONE
;
320 if (ir
->o
== IR_KNUM
) {
321 RegSet avail
= as
->freeset
& ~as
->modset
& RSET_FPR
;
322 lua_assert(allow
!= RSET_EMPTY
);
323 if (!(avail
& (avail
-1))) { /* Fuse if less than two regs available. */
324 as
->mrm
.ofs
= ptr2addr(ir_knum(ir
));
325 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
328 } else if (mayfuse(as
, ref
)) {
329 RegSet xallow
= (allow
& RSET_GPR
) ? allow
: RSET_GPR
;
330 if (ir
->o
== IR_SLOAD
) {
331 if (!(ir
->op2
& (IRSLOAD_PARENT
|IRSLOAD_CONVERT
)) &&
332 noconflict(as
, ref
, IR_RETF
, 0)) {
333 as
->mrm
.base
= (uint8_t)ra_alloc1(as
, REF_BASE
, xallow
);
334 as
->mrm
.ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
&IRSLOAD_FRAME
)?4:0);
335 as
->mrm
.idx
= RID_NONE
;
338 } else if (ir
->o
== IR_FLOAD
) {
339 /* Generic fusion is only ok for 32 bit operand (but see asm_comp). */
340 if ((irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
)) &&
341 noconflict(as
, ref
, IR_FSTORE
, 0)) {
342 asm_fusefref(as
, ir
, xallow
);
345 } else if (ir
->o
== IR_ALOAD
|| ir
->o
== IR_HLOAD
|| ir
->o
== IR_ULOAD
) {
346 if (noconflict(as
, ref
, ir
->o
+ IRDELTA_L2S
, 0)) {
347 asm_fuseahuref(as
, ir
->op1
, xallow
);
350 } else if (ir
->o
== IR_XLOAD
) {
351 /* Generic fusion is not ok for 8/16 bit operands (but see asm_comp).
352 ** Fusing unaligned memory operands is ok on x86 (except for SIMD types).
354 if ((!irt_typerange(ir
->t
, IRT_I8
, IRT_U16
)) &&
355 noconflict(as
, ref
, IR_XSTORE
, 0)) {
356 asm_fusexref(as
, ir
->op1
, xallow
);
359 } else if (ir
->o
== IR_VLOAD
) {
360 asm_fuseahuref(as
, ir
->op1
, xallow
);
364 if (!(as
->freeset
& allow
) &&
365 (allow
== RSET_EMPTY
|| ra_hasspill(ir
->s
) || iscrossref(as
, ref
)))
367 return ra_allocref(as
, ref
, allow
);
370 /* -- Calls --------------------------------------------------------------- */
372 /* Count the required number of stack slots for a call. */
373 static int asm_count_call_slots(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
375 uint32_t i
, nargs
= CCI_NARGS(ci
);
379 nslots
= (int)(nargs
*2); /* Only matters for more than four args. */
381 int ngpr
= REGARG_NUMGPR
, nfpr
= REGARG_NUMFPR
;
382 for (i
= 0; i
< nargs
; i
++)
383 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
384 if (nfpr
> 0) nfpr
--; else nslots
+= 2;
386 if (ngpr
> 0) ngpr
--; else nslots
+= 2;
391 if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_FASTCALL
)
393 else if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_THISCALL
)
395 for (i
= 0; i
< nargs
; i
++)
396 if (args
[i
] && irt_isfp(IR(args
[i
])->t
)) {
397 nslots
+= irt_isnum(IR(args
[i
])->t
) ? 2 : 1;
399 if (ngpr
> 0) ngpr
--; else nslots
++;
405 /* Generate a call to a C function. */
406 static void asm_gencall(ASMState
*as
, const CCallInfo
*ci
, IRRef
*args
)
408 uint32_t n
, nargs
= CCI_NARGS(ci
);
409 int32_t ofs
= STACKARG_OFS
;
411 uint32_t gprs
= REGARG_GPRS
;
412 Reg fpr
= REGARG_FIRSTFPR
;
414 MCode
*patchnfpr
= NULL
;
418 if ((ci
->flags
& CCI_CC_MASK
) != CCI_CC_CDECL
) {
419 if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_THISCALL
)
420 gprs
= (REGARG_GPRS
& 31);
421 else if ((ci
->flags
& CCI_CC_MASK
) == CCI_CC_FASTCALL
)
425 if ((void *)ci
->func
)
426 emit_call(as
, ci
->func
);
428 if ((ci
->flags
& CCI_VARARG
)) { /* Special handling for vararg calls. */
430 for (n
= 0; n
< 4 && n
< nargs
; n
++) {
431 IRIns
*ir
= IR(args
[n
]);
432 if (irt_isfp(ir
->t
)) /* Duplicate FPRs in GPRs. */
433 emit_rr(as
, XO_MOVDto
, (irt_isnum(ir
->t
) ? REX_64
: 0) | (fpr
+n
),
434 ((gprs
>> (n
*5)) & 31)); /* Either MOVD or MOVQ. */
437 patchnfpr
= --as
->mcp
; /* Indicate number of used FPRs in register al. */
438 *--as
->mcp
= XI_MOVrib
| RID_EAX
;
442 for (n
= 0; n
< nargs
; n
++) { /* Setup args. */
446 #if LJ_64 && LJ_ABI_WIN
447 /* Windows/x64 argument registers are strictly positional. */
448 r
= irt_isfp(ir
->t
) ? (fpr
<= REGARG_LASTFPR
? fpr
: 0) : (gprs
& 31);
451 /* POSIX/x64 argument registers are used in order of appearance. */
452 if (irt_isfp(ir
->t
)) {
453 r
= fpr
<= REGARG_LASTFPR
? fpr
++ : 0;
455 r
= gprs
& 31; gprs
>>= 5;
458 if (ref
&& irt_isfp(ir
->t
)) {
461 r
= gprs
& 31; gprs
>>= 5;
465 if (r
) { /* Argument is in a register. */
466 if (r
< RID_MAX_GPR
&& ref
< ASMREF_TMP1
) {
468 if (ir
->o
== IR_KINT64
)
469 emit_loadu64(as
, r
, ir_kint64(ir
)->u64
);
472 emit_loadi(as
, r
, ir
->i
);
474 lua_assert(rset_test(as
->freeset
, r
)); /* Must have been evicted. */
475 if (ra_hasreg(ir
->r
)) {
476 ra_noweak(as
, ir
->r
);
477 emit_movrr(as
, ir
, r
, ir
->r
);
479 ra_allocref(as
, ref
, RID2RSET(r
));
482 } else if (irt_isfp(ir
->t
)) { /* FP argument is on stack. */
483 lua_assert(!(irt_isfloat(ir
->t
) && irref_isk(ref
))); /* No float k. */
484 if (LJ_32
&& (ofs
& 4) && irref_isk(ref
)) {
485 /* Split stores for unaligned FP consts. */
486 emit_movmroi(as
, RID_ESP
, ofs
, (int32_t)ir_knum(ir
)->u32
.lo
);
487 emit_movmroi(as
, RID_ESP
, ofs
+4, (int32_t)ir_knum(ir
)->u32
.hi
);
489 r
= ra_alloc1(as
, ref
, RSET_FPR
);
490 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_MOVSDto
: XO_MOVSSto
,
493 ofs
+= (LJ_32
&& irt_isfloat(ir
->t
)) ? 4 : 8;
494 } else { /* Non-FP argument is on stack. */
495 if (LJ_32
&& ref
< ASMREF_TMP1
) {
496 emit_movmroi(as
, RID_ESP
, ofs
, ir
->i
);
498 r
= ra_alloc1(as
, ref
, RSET_GPR
);
499 emit_movtomro(as
, REX_64
+ r
, RID_ESP
, ofs
);
501 ofs
+= sizeof(intptr_t);
504 #if LJ_64 && !LJ_ABI_WIN
505 if (patchnfpr
) *patchnfpr
= fpr
- REGARG_FIRSTFPR
;
509 /* Setup result reg/sp for call. Evict scratch regs. */
510 static void asm_setupresult(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
512 RegSet drop
= RSET_SCRATCH
;
513 if ((ci
->flags
& CCI_NOFPRCLOBBER
))
515 if (ra_hasreg(ir
->r
))
516 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
517 ra_evictset(as
, drop
); /* Evictions must be performed first. */
519 if (irt_isfp(ir
->t
)) {
520 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
522 if ((ci
->flags
& CCI_CASTU64
)) {
524 if (ra_hasreg(dest
)) {
526 ra_modified(as
, dest
);
527 emit_rr(as
, XO_MOVD
, dest
|REX_64
, RID_RET
); /* Really MOVQ. */
529 if (ofs
) emit_movtomro(as
, RID_RET
|REX_64
, RID_ESP
, ofs
);
531 ra_destreg(as
, ir
, RID_FPRET
);
534 /* Number result is in x87 st0 for x86 calling convention. */
536 if (ra_hasreg(dest
)) {
538 ra_modified(as
, dest
);
539 emit_rmro(as
, irt_isnum(ir
->t
) ? XMM_MOVRM(as
) : XO_MOVSS
,
542 if ((ci
->flags
& CCI_CASTU64
)) {
543 emit_movtomro(as
, RID_RETLO
, RID_ESP
, ofs
);
544 emit_movtomro(as
, RID_RETHI
, RID_ESP
, ofs
+4);
546 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_FSTPq
: XO_FSTPd
,
547 irt_isnum(ir
->t
) ? XOg_FSTPq
: XOg_FSTPd
, RID_ESP
, ofs
);
551 lua_assert(!irt_ispri(ir
->t
));
552 ra_destreg(as
, ir
, RID_RET
);
554 } else if (LJ_32
&& irt_isfp(ir
->t
)) {
555 emit_x87op(as
, XI_FPOP
); /* Pop unused result from x87 st0. */
559 static void asm_call(ASMState
*as
, IRIns
*ir
)
561 IRRef args
[CCI_NARGS_MAX
];
562 const CCallInfo
*ci
= &lj_ir_callinfo
[ir
->op2
];
563 asm_collectargs(as
, ir
, ci
, args
);
564 asm_setupresult(as
, ir
, ci
);
565 asm_gencall(as
, ci
, args
);
568 /* Return a constant function pointer or NULL for indirect calls. */
569 static void *asm_callx_func(ASMState
*as
, IRIns
*irf
, IRRef func
)
574 return (void *)irf
->i
;
576 if (irref_isk(func
)) {
578 if (irf
->o
== IR_KINT64
)
579 p
= (MCode
*)(void *)ir_k64(irf
)->u64
;
581 p
= (MCode
*)(void *)(uintptr_t)(uint32_t)irf
->i
;
582 if (p
- as
->mcp
== (int32_t)(p
- as
->mcp
))
583 return p
; /* Call target is still in +-2GB range. */
584 /* Avoid the indirect case of emit_call(). Try to hoist func addr. */
590 static void asm_callx(ASMState
*as
, IRIns
*ir
)
592 IRRef args
[CCI_NARGS_MAX
];
597 ci
.flags
= asm_callx_flags(as
, ir
);
598 asm_collectargs(as
, ir
, &ci
, args
);
599 asm_setupresult(as
, ir
, &ci
);
601 /* Have to readjust stack after non-cdecl calls due to callee cleanup. */
602 if ((ci
.flags
& CCI_CC_MASK
) != CCI_CC_CDECL
)
603 spadj
= 4 * asm_count_call_slots(as
, &ci
, args
);
605 func
= ir
->op2
; irf
= IR(func
);
606 if (irf
->o
== IR_CARG
) { func
= irf
->op1
; irf
= IR(func
); }
607 ci
.func
= (ASMFunction
)asm_callx_func(as
, irf
, func
);
608 if (!(void *)ci
.func
) {
609 /* Use a (hoistable) non-scratch register for indirect calls. */
610 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
611 Reg r
= ra_alloc1(as
, func
, allow
);
612 if (LJ_32
) emit_spsub(as
, spadj
); /* Above code may cause restores! */
613 emit_rr(as
, XO_GROUP5
, XOg_CALL
, r
);
615 emit_spsub(as
, spadj
);
617 asm_gencall(as
, &ci
, args
);
620 /* -- Returns ------------------------------------------------------------- */
622 /* Return to lower frame. Guard that it goes to the right spot. */
623 static void asm_retf(ASMState
*as
, IRIns
*ir
)
625 Reg base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
626 void *pc
= ir_kptr(IR(ir
->op2
));
627 int32_t delta
= 1+bc_a(*((const BCIns
*)pc
- 1));
628 as
->topslot
-= (BCReg
)delta
;
629 if ((int32_t)as
->topslot
< 0) as
->topslot
= 0;
630 emit_setgl(as
, base
, jit_base
);
631 emit_addptr(as
, base
, -8*delta
);
632 asm_guardcc(as
, CC_NE
);
633 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), base
, -4, ptr2addr(pc
));
636 /* -- Type conversions ---------------------------------------------------- */
638 static void asm_tointg(ASMState
*as
, IRIns
*ir
, Reg left
)
640 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_FPR
, left
));
641 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
642 asm_guardcc(as
, CC_P
);
643 asm_guardcc(as
, CC_NE
);
644 emit_rr(as
, XO_UCOMISD
, left
, tmp
);
645 emit_rr(as
, XO_CVTSI2SD
, tmp
, dest
);
646 if (!(as
->flags
& JIT_F_SPLIT_XMM
))
647 emit_rr(as
, XO_XORPS
, tmp
, tmp
); /* Avoid partial register stall. */
648 emit_rr(as
, XO_CVTTSD2SI
, dest
, left
);
649 /* Can't fuse since left is needed twice. */
652 static void asm_tobit(ASMState
*as
, IRIns
*ir
)
654 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
655 Reg tmp
= ra_noreg(IR(ir
->op1
)->r
) ?
656 ra_alloc1(as
, ir
->op1
, RSET_FPR
) :
657 ra_scratch(as
, RSET_FPR
);
658 Reg right
= asm_fuseload(as
, ir
->op2
, rset_exclude(RSET_FPR
, tmp
));
659 emit_rr(as
, XO_MOVDto
, tmp
, dest
);
660 emit_mrm(as
, XO_ADDSD
, tmp
, right
);
661 ra_left(as
, tmp
, ir
->op1
);
664 static void asm_conv(ASMState
*as
, IRIns
*ir
)
666 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
667 int st64
= (st
== IRT_I64
|| st
== IRT_U64
|| (LJ_64
&& st
== IRT_P64
));
668 int stfp
= (st
== IRT_NUM
|| st
== IRT_FLOAT
);
669 IRRef lref
= ir
->op1
;
670 lua_assert(irt_type(ir
->t
) != st
);
671 lua_assert(!(LJ_32
&& (irt_isint64(ir
->t
) || st64
))); /* Handled by SPLIT. */
672 if (irt_isfp(ir
->t
)) {
673 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
674 if (stfp
) { /* FP to FP conversion. */
675 Reg left
= asm_fuseload(as
, lref
, RSET_FPR
);
676 emit_mrm(as
, st
== IRT_NUM
? XO_CVTSD2SS
: XO_CVTSS2SD
, dest
, left
);
677 if (left
== dest
) return; /* Avoid the XO_XORPS. */
678 } else if (LJ_32
&& st
== IRT_U32
) { /* U32 to FP conversion on x86. */
679 /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
680 cTValue
*k
= lj_ir_k64_find(as
->J
, U64x(43380000,00000000));
681 Reg bias
= ra_scratch(as
, rset_exclude(RSET_FPR
, dest
));
682 if (irt_isfloat(ir
->t
))
683 emit_rr(as
, XO_CVTSD2SS
, dest
, dest
);
684 emit_rr(as
, XO_SUBSD
, dest
, bias
); /* Subtract 2^52+2^51 bias. */
685 emit_rr(as
, XO_XORPS
, dest
, bias
); /* Merge bias and integer. */
686 emit_loadn(as
, bias
, k
);
687 emit_mrm(as
, XO_MOVD
, dest
, asm_fuseload(as
, lref
, RSET_GPR
));
689 } else { /* Integer to FP conversion. */
690 Reg left
= (LJ_64
&& (st
== IRT_U32
|| st
== IRT_U64
)) ?
691 ra_alloc1(as
, lref
, RSET_GPR
) :
692 asm_fuseload(as
, lref
, RSET_GPR
);
693 if (LJ_64
&& st
== IRT_U64
) {
694 MCLabel l_end
= emit_label(as
);
695 const void *k
= lj_ir_k64_find(as
->J
, U64x(43f00000
,00000000));
696 emit_rma(as
, XO_ADDSD
, dest
, k
); /* Add 2^64 to compensate. */
697 emit_sjcc(as
, CC_NS
, l_end
);
698 emit_rr(as
, XO_TEST
, left
|REX_64
, left
); /* Check if u64 >= 2^63. */
700 emit_mrm(as
, irt_isnum(ir
->t
) ? XO_CVTSI2SD
: XO_CVTSI2SS
,
701 dest
|((LJ_64
&& (st64
|| st
== IRT_U32
)) ? REX_64
: 0), left
);
703 if (!(as
->flags
& JIT_F_SPLIT_XMM
))
704 emit_rr(as
, XO_XORPS
, dest
, dest
); /* Avoid partial register stall. */
705 } else if (stfp
) { /* FP to integer conversion. */
706 if (irt_isguard(ir
->t
)) {
707 /* Checked conversions are only supported from number to int. */
708 lua_assert(irt_isint(ir
->t
) && st
== IRT_NUM
);
709 asm_tointg(as
, ir
, ra_alloc1(as
, lref
, RSET_FPR
));
711 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
712 x86Op op
= st
== IRT_NUM
?
713 ((ir
->op2
& IRCONV_TRUNC
) ? XO_CVTTSD2SI
: XO_CVTSD2SI
) :
714 ((ir
->op2
& IRCONV_TRUNC
) ? XO_CVTTSS2SI
: XO_CVTSS2SI
);
715 if (LJ_64
? irt_isu64(ir
->t
) : irt_isu32(ir
->t
)) {
716 /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */
717 /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */
718 Reg tmp
= ra_noreg(IR(lref
)->r
) ? ra_alloc1(as
, lref
, RSET_FPR
) :
719 ra_scratch(as
, RSET_FPR
);
720 MCLabel l_end
= emit_label(as
);
722 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, (int32_t)0x80000000);
723 emit_rr(as
, op
, dest
|REX_64
, tmp
);
725 emit_rma(as
, XO_ADDSD
, tmp
, lj_ir_k64_find(as
->J
,
726 LJ_64
? U64x(c3f00000
,00000000) : U64x(c1e00000
,00000000)));
728 emit_rma(as
, XO_ADDSS
, tmp
, lj_ir_k64_find(as
->J
,
729 LJ_64
? U64x(00000000,df800000
) : U64x(00000000,cf000000
)));
730 emit_sjcc(as
, CC_NS
, l_end
);
731 emit_rr(as
, XO_TEST
, dest
|REX_64
, dest
); /* Check if dest negative. */
732 emit_rr(as
, op
, dest
|REX_64
, tmp
);
733 ra_left(as
, tmp
, lref
);
735 Reg left
= asm_fuseload(as
, lref
, RSET_FPR
);
736 if (LJ_64
&& irt_isu32(ir
->t
))
737 emit_rr(as
, XO_MOV
, dest
, dest
); /* Zero hiword. */
740 (irt_is64(ir
->t
) || irt_isu32(ir
->t
))) ? REX_64
: 0),
744 } else if (st
>= IRT_I8
&& st
<= IRT_U16
) { /* Extend to 32 bit integer. */
745 Reg left
, dest
= ra_dest(as
, ir
, RSET_GPR
);
746 RegSet allow
= RSET_GPR
;
748 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
));
750 op
= XO_MOVSXb
; allow
= RSET_GPR8
; dest
|= FORCE_REX
;
751 } else if (st
== IRT_U8
) {
752 op
= XO_MOVZXb
; allow
= RSET_GPR8
; dest
|= FORCE_REX
;
753 } else if (st
== IRT_I16
) {
758 left
= asm_fuseload(as
, lref
, allow
);
759 /* Add extra MOV if source is already in wrong register. */
760 if (!LJ_64
&& left
!= RID_MRM
&& !rset_test(allow
, left
)) {
761 Reg tmp
= ra_scratch(as
, allow
);
762 emit_rr(as
, op
, dest
, tmp
);
763 emit_rr(as
, XO_MOV
, tmp
, left
);
765 emit_mrm(as
, op
, dest
, left
);
767 } else { /* 32/64 bit integer conversions. */
768 if (LJ_32
) { /* Only need to handle 32/32 bit no-op (cast) on x86. */
769 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
770 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
771 } else if (irt_is64(ir
->t
)) {
772 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
773 if (st64
|| !(ir
->op2
& IRCONV_SEXT
)) {
774 /* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
775 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
776 } else { /* 32 to 64 bit sign extension. */
777 Reg left
= asm_fuseload(as
, lref
, RSET_GPR
);
778 emit_mrm(as
, XO_MOVSXd
, dest
|REX_64
, left
);
781 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
783 Reg left
= asm_fuseload(as
, lref
, RSET_GPR
);
784 /* This is either a 32 bit reg/reg mov which zeroes the hiword
785 ** or a load of the loword from a 64 bit address.
787 emit_mrm(as
, XO_MOV
, dest
, left
);
788 } else { /* 32/32 bit no-op (cast). */
789 ra_left(as
, dest
, lref
); /* Do nothing, but may need to move regs. */
795 #if LJ_32 && LJ_HASFFI
796 /* No SSE conversions to/from 64 bit on x86, so resort to ugly x87 code. */
798 /* 64 bit integer to FP conversion in 32 bit mode. */
799 static void asm_conv_fp_int64(ASMState
*as
, IRIns
*ir
)
801 Reg hi
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
802 Reg lo
= ra_alloc1(as
, (ir
-1)->op1
, rset_exclude(RSET_GPR
, hi
));
803 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
805 if (ra_hasreg(dest
)) {
807 ra_modified(as
, dest
);
808 emit_rmro(as
, irt_isnum(ir
->t
) ? XMM_MOVRM(as
) : XO_MOVSS
,
811 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_FSTPq
: XO_FSTPd
,
812 irt_isnum(ir
->t
) ? XOg_FSTPq
: XOg_FSTPd
, RID_ESP
, ofs
);
813 if (((ir
-1)->op2
& IRCONV_SRCMASK
) == IRT_U64
) {
814 /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
815 MCLabel l_end
= emit_label(as
);
816 emit_rma(as
, XO_FADDq
, XOg_FADDq
,
817 lj_ir_k64_find(as
->J
, U64x(43f00000
,00000000)));
818 emit_sjcc(as
, CC_NS
, l_end
);
819 emit_rr(as
, XO_TEST
, hi
, hi
); /* Check if u64 >= 2^63. */
821 lua_assert(((ir
-1)->op2
& IRCONV_SRCMASK
) == IRT_I64
);
823 emit_rmro(as
, XO_FILDq
, XOg_FILDq
, RID_ESP
, 0);
824 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
825 emit_rmro(as
, XO_MOVto
, hi
, RID_ESP
, 4);
826 emit_rmro(as
, XO_MOVto
, lo
, RID_ESP
, 0);
829 /* FP to 64 bit integer conversion in 32 bit mode. */
830 static void asm_conv_int64_fp(ASMState
*as
, IRIns
*ir
)
832 IRType st
= (IRType
)((ir
-1)->op2
& IRCONV_SRCMASK
);
833 IRType dt
= (((ir
-1)->op2
& IRCONV_DSTMASK
) >> IRCONV_DSH
);
835 lua_assert(st
== IRT_NUM
|| st
== IRT_FLOAT
);
836 lua_assert(dt
== IRT_I64
|| dt
== IRT_U64
);
837 lua_assert(((ir
-1)->op2
& IRCONV_TRUNC
));
838 hi
= ra_dest(as
, ir
, RSET_GPR
);
839 lo
= ra_dest(as
, ir
-1, rset_exclude(RSET_GPR
, hi
));
840 if (ra_used(ir
-1)) emit_rmro(as
, XO_MOV
, lo
, RID_ESP
, 0);
841 /* NYI: Avoid wide-to-narrow store-to-load forwarding stall. */
842 if (!(as
->flags
& JIT_F_SSE3
)) { /* Set FPU rounding mode to default. */
843 emit_rmro(as
, XO_FLDCW
, XOg_FLDCW
, RID_ESP
, 4);
844 emit_rmro(as
, XO_MOVto
, lo
, RID_ESP
, 4);
845 emit_gri(as
, XG_ARITHi(XOg_AND
), lo
, 0xf3ff);
848 /* For inputs in [2^63,2^64-1] add -2^64 and convert again. */
849 MCLabel l_pop
, l_end
= emit_label(as
);
850 emit_x87op(as
, XI_FPOP
);
851 l_pop
= emit_label(as
);
852 emit_sjmp(as
, l_end
);
853 emit_rmro(as
, XO_MOV
, hi
, RID_ESP
, 4);
854 if ((as
->flags
& JIT_F_SSE3
))
855 emit_rmro(as
, XO_FISTTPq
, XOg_FISTTPq
, RID_ESP
, 0);
857 emit_rmro(as
, XO_FISTPq
, XOg_FISTPq
, RID_ESP
, 0);
858 emit_rma(as
, XO_FADDq
, XOg_FADDq
,
859 lj_ir_k64_find(as
->J
, U64x(c3f00000
,00000000)));
860 emit_sjcc(as
, CC_NS
, l_pop
);
861 emit_rr(as
, XO_TEST
, hi
, hi
); /* Check if out-of-range (2^63). */
863 emit_rmro(as
, XO_MOV
, hi
, RID_ESP
, 4);
864 if ((as
->flags
& JIT_F_SSE3
)) { /* Truncation is easy with SSE3. */
865 emit_rmro(as
, XO_FISTTPq
, XOg_FISTTPq
, RID_ESP
, 0);
866 } else { /* Otherwise set FPU rounding mode to truncate before the store. */
867 emit_rmro(as
, XO_FISTPq
, XOg_FISTPq
, RID_ESP
, 0);
868 emit_rmro(as
, XO_FLDCW
, XOg_FLDCW
, RID_ESP
, 0);
869 emit_rmro(as
, XO_MOVtow
, lo
, RID_ESP
, 0);
870 emit_rmro(as
, XO_ARITHw(XOg_OR
), lo
, RID_ESP
, 0);
871 emit_loadi(as
, lo
, 0xc00);
872 emit_rmro(as
, XO_FNSTCW
, XOg_FNSTCW
, RID_ESP
, 0);
875 emit_x87op(as
, XI_FDUP
);
876 emit_mrm(as
, st
== IRT_NUM
? XO_FLDq
: XO_FLDd
,
877 st
== IRT_NUM
? XOg_FLDq
: XOg_FLDd
,
878 asm_fuseload(as
, ir
->op1
, RSET_EMPTY
));
882 static void asm_strto(ASMState
*as
, IRIns
*ir
)
884 /* Force a spill slot for the destination register (if any). */
885 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_strscan_num
];
887 RegSet drop
= RSET_SCRATCH
;
888 if ((drop
& RSET_FPR
) != RSET_FPR
&& ra_hasreg(ir
->r
))
889 rset_set(drop
, ir
->r
); /* WIN64 doesn't spill all FPRs. */
890 ra_evictset(as
, drop
);
891 asm_guardcc(as
, CC_E
);
892 emit_rr(as
, XO_TEST
, RID_RET
, RID_RET
); /* Test return status. */
893 args
[0] = ir
->op1
; /* GCstr *str */
894 args
[1] = ASMREF_TMP1
; /* TValue *n */
895 asm_gencall(as
, ci
, args
);
896 /* Store the result to the spill slot or temp slots. */
897 emit_rmro(as
, XO_LEA
, ra_releasetmp(as
, ASMREF_TMP1
)|REX_64
,
898 RID_ESP
, sps_scale(ir
->s
));
901 static void asm_tostr(ASMState
*as
, IRIns
*ir
)
903 IRIns
*irl
= IR(ir
->op1
);
907 if (irt_isnum(irl
->t
)) {
908 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromnum
];
909 args
[1] = ASMREF_TMP1
; /* const lua_Number * */
910 asm_setupresult(as
, ir
, ci
); /* GCstr * */
911 asm_gencall(as
, ci
, args
);
912 emit_rmro(as
, XO_LEA
, ra_releasetmp(as
, ASMREF_TMP1
)|REX_64
,
913 RID_ESP
, ra_spill(as
, irl
));
915 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_str_fromint
];
916 args
[1] = ir
->op1
; /* int32_t k */
917 asm_setupresult(as
, ir
, ci
); /* GCstr * */
918 asm_gencall(as
, ci
, args
);
922 /* -- Memory references --------------------------------------------------- */
924 static void asm_aref(ASMState
*as
, IRIns
*ir
)
926 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
927 asm_fusearef(as
, ir
, RSET_GPR
);
928 if (!(as
->mrm
.idx
== RID_NONE
&& as
->mrm
.ofs
== 0))
929 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
930 else if (as
->mrm
.base
!= dest
)
931 emit_rr(as
, XO_MOV
, dest
, as
->mrm
.base
);
934 /* Merge NE(HREF, niltv) check. */
935 static MCode
*merge_href_niltv(ASMState
*as
, IRIns
*ir
)
937 /* Assumes nothing else generates NE of HREF. */
938 if ((ir
[1].o
== IR_NE
|| ir
[1].o
== IR_EQ
) && ir
[1].op1
== as
->curins
&&
941 p
+= (LJ_64
&& *p
!= XI_ARITHi
) ? 7+6 : 6+6;
942 /* Ensure no loop branch inversion happened. */
943 if (p
[-6] == 0x0f && p
[-5] == XI_JCCn
+(CC_NE
^(ir
[1].o
& 1))) {
944 as
->mcp
= p
; /* Kill cmp reg, imm32 + jz exit. */
945 return p
+ *(int32_t *)(p
-4); /* Return exit address. */
951 /* Inlined hash lookup. Specialized for key type and for const keys.
952 ** The equivalent C code is:
953 ** Node *n = hashkey(t, key);
955 ** if (lj_obj_equal(&n->key, key)) return &n->val;
956 ** } while ((n = nextnode(n)));
959 static void asm_href(ASMState
*as
, IRIns
*ir
)
961 MCode
*nilexit
= merge_href_niltv(as
, ir
); /* Do this before any restores. */
962 RegSet allow
= RSET_GPR
;
963 Reg dest
= ra_dest(as
, ir
, allow
);
964 Reg tab
= ra_alloc1(as
, ir
->op1
, rset_clear(allow
, dest
));
965 Reg key
= RID_NONE
, tmp
= RID_NONE
;
966 IRIns
*irkey
= IR(ir
->op2
);
967 int isk
= irref_isk(ir
->op2
);
968 IRType1 kt
= irkey
->t
;
970 MCLabel l_end
, l_loop
, l_next
;
973 rset_clear(allow
, tab
);
974 key
= ra_alloc1(as
, ir
->op2
, irt_isnum(kt
) ? RSET_FPR
: allow
);
976 tmp
= ra_scratch(as
, rset_exclude(allow
, key
));
979 /* Key not found in chain: jump to exit (if merged with NE) or load niltv. */
980 l_end
= emit_label(as
);
981 if (nilexit
&& ir
[1].o
== IR_NE
) {
982 emit_jcc(as
, CC_E
, nilexit
); /* XI_JMP is not found by lj_asm_patchexit. */
985 emit_loada(as
, dest
, niltvg(J2G(as
->J
)));
988 /* Follow hash chain until the end. */
989 l_loop
= emit_sjcc_label(as
, CC_NZ
);
990 emit_rr(as
, XO_TEST
, dest
, dest
);
991 emit_rmro(as
, XO_MOV
, dest
, dest
, offsetof(Node
, next
));
992 l_next
= emit_label(as
);
994 /* Type and value comparison. */
996 emit_jcc(as
, CC_E
, nilexit
);
998 emit_sjcc(as
, CC_E
, l_end
);
1001 /* Assumes -0.0 is already canonicalized to +0.0. */
1002 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.u32
.lo
),
1003 (int32_t)ir_knum(irkey
)->u32
.lo
);
1004 emit_sjcc(as
, CC_NE
, l_next
);
1005 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.u32
.hi
),
1006 (int32_t)ir_knum(irkey
)->u32
.hi
);
1008 emit_sjcc(as
, CC_P
, l_next
);
1009 emit_rmro(as
, XO_UCOMISD
, key
, dest
, offsetof(Node
, key
.n
));
1010 emit_sjcc(as
, CC_AE
, l_next
);
1011 /* The type check avoids NaN penalties and complaints from Valgrind. */
1013 emit_u32(as
, LJ_TISNUM
);
1014 emit_rmro(as
, XO_ARITHi
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1016 emit_i8(as
, LJ_TISNUM
);
1017 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1021 } else if (irt_islightud(kt
)) {
1022 emit_rmro(as
, XO_CMP
, key
|REX_64
, dest
, offsetof(Node
, key
.u64
));
1025 if (!irt_ispri(kt
)) {
1026 lua_assert(irt_isaddr(kt
));
1028 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), dest
, offsetof(Node
, key
.gcr
),
1029 ptr2addr(ir_kgc(irkey
)));
1031 emit_rmro(as
, XO_CMP
, key
, dest
, offsetof(Node
, key
.gcr
));
1032 emit_sjcc(as
, CC_NE
, l_next
);
1034 lua_assert(!irt_isnil(kt
));
1035 emit_i8(as
, irt_toitype(kt
));
1036 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, dest
, offsetof(Node
, key
.it
));
1038 emit_sfixup(as
, l_loop
);
1041 /* Load main position relative to tab->node into dest. */
1042 khash
= isk
? ir_khash(irkey
) : 1;
1044 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, node
));
1046 emit_rmro(as
, XO_ARITH(XOg_ADD
), dest
, tab
, offsetof(GCtab
, node
));
1047 if ((as
->flags
& JIT_F_PREFER_IMUL
)) {
1048 emit_i8(as
, sizeof(Node
));
1049 emit_rr(as
, XO_IMULi8
, dest
, dest
);
1051 emit_shifti(as
, XOg_SHL
, dest
, 3);
1052 emit_rmrxo(as
, XO_LEA
, dest
, dest
, dest
, XM_SCALE2
, 0);
1055 emit_gri(as
, XG_ARITHi(XOg_AND
), dest
, (int32_t)khash
);
1056 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, hmask
));
1057 } else if (irt_isstr(kt
)) {
1058 emit_rmro(as
, XO_ARITH(XOg_AND
), dest
, key
, offsetof(GCstr
, hash
));
1059 emit_rmro(as
, XO_MOV
, dest
, tab
, offsetof(GCtab
, hmask
));
1060 } else { /* Must match with hashrot() in lj_tab.c. */
1061 emit_rmro(as
, XO_ARITH(XOg_AND
), dest
, tab
, offsetof(GCtab
, hmask
));
1062 emit_rr(as
, XO_ARITH(XOg_SUB
), dest
, tmp
);
1063 emit_shifti(as
, XOg_ROL
, tmp
, HASH_ROT3
);
1064 emit_rr(as
, XO_ARITH(XOg_XOR
), dest
, tmp
);
1065 emit_shifti(as
, XOg_ROL
, dest
, HASH_ROT2
);
1066 emit_rr(as
, XO_ARITH(XOg_SUB
), tmp
, dest
);
1067 emit_shifti(as
, XOg_ROL
, dest
, HASH_ROT1
);
1068 emit_rr(as
, XO_ARITH(XOg_XOR
), tmp
, dest
);
1069 if (irt_isnum(kt
)) {
1070 emit_rr(as
, XO_ARITH(XOg_ADD
), dest
, dest
);
1072 emit_shifti(as
, XOg_SHR
|REX_64
, dest
, 32);
1073 emit_rr(as
, XO_MOV
, tmp
, dest
);
1074 emit_rr(as
, XO_MOVDto
, key
|REX_64
, dest
);
1076 emit_rmro(as
, XO_MOV
, dest
, RID_ESP
, ra_spill(as
, irkey
)+4);
1077 emit_rr(as
, XO_MOVDto
, key
, tmp
);
1080 emit_rr(as
, XO_MOV
, tmp
, key
);
1081 emit_rmro(as
, XO_LEA
, dest
, key
, HASH_BIAS
);
1087 static void asm_hrefk(ASMState
*as
, IRIns
*ir
)
1089 IRIns
*kslot
= IR(ir
->op2
);
1090 IRIns
*irkey
= IR(kslot
->op1
);
1091 int32_t ofs
= (int32_t)(kslot
->op2
* sizeof(Node
));
1092 Reg dest
= ra_used(ir
) ? ra_dest(as
, ir
, RSET_GPR
) : RID_NONE
;
1093 Reg node
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1097 lua_assert(ofs
% sizeof(Node
) == 0);
1098 if (ra_hasreg(dest
)) {
1100 if (dest
== node
&& !(as
->flags
& JIT_F_LEA_AGU
))
1101 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, ofs
);
1103 emit_rmro(as
, XO_LEA
, dest
, node
, ofs
);
1104 } else if (dest
!= node
) {
1105 emit_rr(as
, XO_MOV
, dest
, node
);
1108 asm_guardcc(as
, CC_NE
);
1110 if (!irt_ispri(irkey
->t
)) {
1111 Reg key
= ra_scratch(as
, rset_exclude(RSET_GPR
, node
));
1112 emit_rmro(as
, XO_CMP
, key
|REX_64
, node
,
1113 ofs
+ (int32_t)offsetof(Node
, key
.u64
));
1114 lua_assert(irt_isnum(irkey
->t
) || irt_isgcv(irkey
->t
));
1115 /* Assumes -0.0 is already canonicalized to +0.0. */
1116 emit_loadu64(as
, key
, irt_isnum(irkey
->t
) ? ir_knum(irkey
)->u64
:
1117 ((uint64_t)irt_toitype(irkey
->t
) << 32) |
1118 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey
)));
1120 lua_assert(!irt_isnil(irkey
->t
));
1121 emit_i8(as
, irt_toitype(irkey
->t
));
1122 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, node
,
1123 ofs
+ (int32_t)offsetof(Node
, key
.it
));
1126 l_exit
= emit_label(as
);
1127 if (irt_isnum(irkey
->t
)) {
1128 /* Assumes -0.0 is already canonicalized to +0.0. */
1129 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1130 ofs
+ (int32_t)offsetof(Node
, key
.u32
.lo
),
1131 (int32_t)ir_knum(irkey
)->u32
.lo
);
1132 emit_sjcc(as
, CC_NE
, l_exit
);
1133 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1134 ofs
+ (int32_t)offsetof(Node
, key
.u32
.hi
),
1135 (int32_t)ir_knum(irkey
)->u32
.hi
);
1137 if (!irt_ispri(irkey
->t
)) {
1138 lua_assert(irt_isgcv(irkey
->t
));
1139 emit_gmroi(as
, XG_ARITHi(XOg_CMP
), node
,
1140 ofs
+ (int32_t)offsetof(Node
, key
.gcr
),
1141 ptr2addr(ir_kgc(irkey
)));
1142 emit_sjcc(as
, CC_NE
, l_exit
);
1144 lua_assert(!irt_isnil(irkey
->t
));
1145 emit_i8(as
, irt_toitype(irkey
->t
));
1146 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, node
,
1147 ofs
+ (int32_t)offsetof(Node
, key
.it
));
1152 static void asm_newref(ASMState
*as
, IRIns
*ir
)
1154 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_tab_newkey
];
1158 if (ir
->r
== RID_SINK
)
1160 args
[0] = ASMREF_L
; /* lua_State *L */
1161 args
[1] = ir
->op1
; /* GCtab *t */
1162 args
[2] = ASMREF_TMP1
; /* cTValue *key */
1163 asm_setupresult(as
, ir
, ci
); /* TValue * */
1164 asm_gencall(as
, ci
, args
);
1165 tmp
= ra_releasetmp(as
, ASMREF_TMP1
);
1166 irkey
= IR(ir
->op2
);
1167 if (irt_isnum(irkey
->t
)) {
1168 /* For numbers use the constant itself or a spill slot as a TValue. */
1169 if (irref_isk(ir
->op2
))
1170 emit_loada(as
, tmp
, ir_knum(irkey
));
1172 emit_rmro(as
, XO_LEA
, tmp
|REX_64
, RID_ESP
, ra_spill(as
, irkey
));
1174 /* Otherwise use g->tmptv to hold the TValue. */
1175 if (!irref_isk(ir
->op2
)) {
1176 Reg src
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_GPR
, tmp
));
1177 emit_movtomro(as
, REX_64IR(irkey
, src
), tmp
, 0);
1178 } else if (!irt_ispri(irkey
->t
)) {
1179 emit_movmroi(as
, tmp
, 0, irkey
->i
);
1181 if (!(LJ_64
&& irt_islightud(irkey
->t
)))
1182 emit_movmroi(as
, tmp
, 4, irt_toitype(irkey
->t
));
1183 emit_loada(as
, tmp
, &J2G(as
->J
)->tmptv
);
1187 static void asm_uref(ASMState
*as
, IRIns
*ir
)
1189 /* NYI: Check that UREFO is still open and not aliasing a slot. */
1190 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1191 if (irref_isk(ir
->op1
)) {
1192 GCfunc
*fn
= ir_kfunc(IR(ir
->op1
));
1193 MRef
*v
= &gcref(fn
->l
.uvptr
[(ir
->op2
>> 8)])->uv
.v
;
1194 emit_rma(as
, XO_MOV
, dest
, v
);
1196 Reg uv
= ra_scratch(as
, RSET_GPR
);
1197 Reg func
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1198 if (ir
->o
== IR_UREFC
) {
1199 emit_rmro(as
, XO_LEA
, dest
, uv
, offsetof(GCupval
, tv
));
1200 asm_guardcc(as
, CC_NE
);
1202 emit_rmro(as
, XO_ARITHib
, XOg_CMP
, uv
, offsetof(GCupval
, closed
));
1204 emit_rmro(as
, XO_MOV
, dest
, uv
, offsetof(GCupval
, v
));
1206 emit_rmro(as
, XO_MOV
, uv
, func
,
1207 (int32_t)offsetof(GCfuncL
, uvptr
) + 4*(int32_t)(ir
->op2
>> 8));
1211 static void asm_fref(ASMState
*as
, IRIns
*ir
)
1213 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1214 asm_fusefref(as
, ir
, RSET_GPR
);
1215 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1218 static void asm_strref(ASMState
*as
, IRIns
*ir
)
1220 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1221 asm_fusestrref(as
, ir
, RSET_GPR
);
1222 if (as
->mrm
.base
== RID_NONE
)
1223 emit_loadi(as
, dest
, as
->mrm
.ofs
);
1224 else if (as
->mrm
.base
== dest
&& as
->mrm
.idx
== RID_NONE
)
1225 emit_gri(as
, XG_ARITHi(XOg_ADD
), dest
, as
->mrm
.ofs
);
1227 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1230 /* -- Loads and stores ---------------------------------------------------- */
1232 static void asm_fxload(ASMState
*as
, IRIns
*ir
)
1234 Reg dest
= ra_dest(as
, ir
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
);
1236 if (ir
->o
== IR_FLOAD
)
1237 asm_fusefref(as
, ir
, RSET_GPR
);
1239 asm_fusexref(as
, ir
->op1
, RSET_GPR
);
1240 /* ir->op2 is ignored -- unaligned loads are ok on x86. */
1241 switch (irt_type(ir
->t
)) {
1242 case IRT_I8
: xo
= XO_MOVSXb
; break;
1243 case IRT_U8
: xo
= XO_MOVZXb
; break;
1244 case IRT_I16
: xo
= XO_MOVSXw
; break;
1245 case IRT_U16
: xo
= XO_MOVZXw
; break;
1246 case IRT_NUM
: xo
= XMM_MOVRM(as
); break;
1247 case IRT_FLOAT
: xo
= XO_MOVSS
; break;
1249 if (LJ_64
&& irt_is64(ir
->t
))
1252 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
));
1256 emit_mrm(as
, xo
, dest
, RID_MRM
);
1259 static void asm_fxstore(ASMState
*as
, IRIns
*ir
)
1261 RegSet allow
= RSET_GPR
;
1262 Reg src
= RID_NONE
, osrc
= RID_NONE
;
1264 if (ir
->r
== RID_SINK
)
1266 /* The IRT_I16/IRT_U16 stores should never be simplified for constant
1267 ** values since mov word [mem], imm16 has a length-changing prefix.
1269 if (irt_isi16(ir
->t
) || irt_isu16(ir
->t
) || irt_isfp(ir
->t
) ||
1270 !asm_isk32(as
, ir
->op2
, &k
)) {
1271 RegSet allow8
= irt_isfp(ir
->t
) ? RSET_FPR
:
1272 (irt_isi8(ir
->t
) || irt_isu8(ir
->t
)) ? RSET_GPR8
: RSET_GPR
;
1273 src
= osrc
= ra_alloc1(as
, ir
->op2
, allow8
);
1274 if (!LJ_64
&& !rset_test(allow8
, src
)) { /* Already in wrong register. */
1275 rset_clear(allow
, osrc
);
1276 src
= ra_scratch(as
, allow8
);
1278 rset_clear(allow
, src
);
1280 if (ir
->o
== IR_FSTORE
) {
1281 asm_fusefref(as
, IR(ir
->op1
), allow
);
1283 asm_fusexref(as
, ir
->op1
, allow
);
1284 if (LJ_32
&& ir
->o
== IR_HIOP
) as
->mrm
.ofs
+= 4;
1286 if (ra_hasreg(src
)) {
1288 switch (irt_type(ir
->t
)) {
1289 case IRT_I8
: case IRT_U8
: xo
= XO_MOVtob
; src
|= FORCE_REX
; break;
1290 case IRT_I16
: case IRT_U16
: xo
= XO_MOVtow
; break;
1291 case IRT_NUM
: xo
= XO_MOVSDto
; break;
1292 case IRT_FLOAT
: xo
= XO_MOVSSto
; break;
1294 case IRT_LIGHTUD
: lua_assert(0); /* NYI: mask 64 bit lightuserdata. */
1297 if (LJ_64
&& irt_is64(ir
->t
))
1300 lua_assert(irt_isint(ir
->t
) || irt_isu32(ir
->t
) || irt_isaddr(ir
->t
));
1304 emit_mrm(as
, xo
, src
, RID_MRM
);
1305 if (!LJ_64
&& src
!= osrc
) {
1306 ra_noweak(as
, osrc
);
1307 emit_rr(as
, XO_MOV
, src
, osrc
);
1310 if (irt_isi8(ir
->t
) || irt_isu8(ir
->t
)) {
1312 emit_mrm(as
, XO_MOVmib
, 0, RID_MRM
);
1314 lua_assert(irt_is64(ir
->t
) || irt_isint(ir
->t
) || irt_isu32(ir
->t
) ||
1317 emit_mrm(as
, XO_MOVmi
, REX_64IR(ir
, 0), RID_MRM
);
1323 static Reg
asm_load_lightud64(ASMState
*as
, IRIns
*ir
, int typecheck
)
1325 if (ra_used(ir
) || typecheck
) {
1326 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1328 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, dest
));
1329 asm_guardcc(as
, CC_NE
);
1331 emit_rr(as
, XO_ARITHi8
, XOg_CMP
, tmp
);
1332 emit_shifti(as
, XOg_SAR
|REX_64
, tmp
, 47);
1333 emit_rr(as
, XO_MOV
, tmp
|REX_64
, dest
);
1342 static void asm_ahuvload(ASMState
*as
, IRIns
*ir
)
1344 lua_assert(irt_isnum(ir
->t
) || irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) ||
1345 (LJ_DUALNUM
&& irt_isint(ir
->t
)));
1347 if (irt_islightud(ir
->t
)) {
1348 Reg dest
= asm_load_lightud64(as
, ir
, 1);
1349 if (ra_hasreg(dest
)) {
1350 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1351 emit_mrm(as
, XO_MOV
, dest
|REX_64
, RID_MRM
);
1357 RegSet allow
= irt_isnum(ir
->t
) ? RSET_FPR
: RSET_GPR
;
1358 Reg dest
= ra_dest(as
, ir
, allow
);
1359 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1360 emit_mrm(as
, dest
< RID_MAX_GPR
? XO_MOV
: XMM_MOVRM(as
), dest
, RID_MRM
);
1362 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1364 /* Always do the type check, even if the load result is unused. */
1366 asm_guardcc(as
, irt_isnum(ir
->t
) ? CC_AE
: CC_NE
);
1367 if (LJ_64
&& irt_type(ir
->t
) >= IRT_NUM
) {
1368 lua_assert(irt_isinteger(ir
->t
) || irt_isnum(ir
->t
));
1369 emit_u32(as
, LJ_TISNUM
);
1370 emit_mrm(as
, XO_ARITHi
, XOg_CMP
, RID_MRM
);
1372 emit_i8(as
, irt_toitype(ir
->t
));
1373 emit_mrm(as
, XO_ARITHi8
, XOg_CMP
, RID_MRM
);
1377 static void asm_ahustore(ASMState
*as
, IRIns
*ir
)
1379 if (ir
->r
== RID_SINK
)
1381 if (irt_isnum(ir
->t
)) {
1382 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_FPR
);
1383 asm_fuseahuref(as
, ir
->op1
, RSET_GPR
);
1384 emit_mrm(as
, XO_MOVSDto
, src
, RID_MRM
);
1386 } else if (irt_islightud(ir
->t
)) {
1387 Reg src
= ra_alloc1(as
, ir
->op2
, RSET_GPR
);
1388 asm_fuseahuref(as
, ir
->op1
, rset_exclude(RSET_GPR
, src
));
1389 emit_mrm(as
, XO_MOVto
, src
|REX_64
, RID_MRM
);
1392 IRIns
*irr
= IR(ir
->op2
);
1393 RegSet allow
= RSET_GPR
;
1395 if (!irref_isk(ir
->op2
)) {
1396 src
= ra_alloc1(as
, ir
->op2
, allow
);
1397 rset_clear(allow
, src
);
1399 asm_fuseahuref(as
, ir
->op1
, allow
);
1400 if (ra_hasreg(src
)) {
1401 emit_mrm(as
, XO_MOVto
, src
, RID_MRM
);
1402 } else if (!irt_ispri(irr
->t
)) {
1403 lua_assert(irt_isaddr(ir
->t
) || (LJ_DUALNUM
&& irt_isinteger(ir
->t
)));
1404 emit_i32(as
, irr
->i
);
1405 emit_mrm(as
, XO_MOVmi
, 0, RID_MRM
);
1408 emit_i32(as
, (int32_t)irt_toitype(ir
->t
));
1409 emit_mrm(as
, XO_MOVmi
, 0, RID_MRM
);
1413 static void asm_sload(ASMState
*as
, IRIns
*ir
)
1415 int32_t ofs
= 8*((int32_t)ir
->op1
-1) + ((ir
->op2
& IRSLOAD_FRAME
) ? 4 : 0);
1418 lua_assert(!(ir
->op2
& IRSLOAD_PARENT
)); /* Handled by asm_head_side(). */
1419 lua_assert(irt_isguard(t
) || !(ir
->op2
& IRSLOAD_TYPECHECK
));
1420 lua_assert(LJ_DUALNUM
||
1421 !irt_isint(t
) || (ir
->op2
& (IRSLOAD_CONVERT
|IRSLOAD_FRAME
)));
1422 if ((ir
->op2
& IRSLOAD_CONVERT
) && irt_isguard(t
) && irt_isint(t
)) {
1423 Reg left
= ra_scratch(as
, RSET_FPR
);
1424 asm_tointg(as
, ir
, left
); /* Frees dest reg. Do this before base alloc. */
1425 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1426 emit_rmro(as
, XMM_MOVRM(as
), left
, base
, ofs
);
1427 t
.irt
= IRT_NUM
; /* Continue with a regular number type check. */
1429 } else if (irt_islightud(t
)) {
1430 Reg dest
= asm_load_lightud64(as
, ir
, (ir
->op2
& IRSLOAD_TYPECHECK
));
1431 if (ra_hasreg(dest
)) {
1432 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1433 emit_rmro(as
, XO_MOV
, dest
|REX_64
, base
, ofs
);
1437 } else if (ra_used(ir
)) {
1438 RegSet allow
= irt_isnum(t
) ? RSET_FPR
: RSET_GPR
;
1439 Reg dest
= ra_dest(as
, ir
, allow
);
1440 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1441 lua_assert(irt_isnum(t
) || irt_isint(t
) || irt_isaddr(t
));
1442 if ((ir
->op2
& IRSLOAD_CONVERT
)) {
1443 t
.irt
= irt_isint(t
) ? IRT_NUM
: IRT_INT
; /* Check for original type. */
1444 emit_rmro(as
, irt_isint(t
) ? XO_CVTSI2SD
: XO_CVTSD2SI
, dest
, base
, ofs
);
1445 } else if (irt_isnum(t
)) {
1446 emit_rmro(as
, XMM_MOVRM(as
), dest
, base
, ofs
);
1448 emit_rmro(as
, XO_MOV
, dest
, base
, ofs
);
1451 if (!(ir
->op2
& IRSLOAD_TYPECHECK
))
1452 return; /* No type check: avoid base alloc. */
1453 base
= ra_alloc1(as
, REF_BASE
, RSET_GPR
);
1455 if ((ir
->op2
& IRSLOAD_TYPECHECK
)) {
1456 /* Need type check, even if the load result is unused. */
1457 asm_guardcc(as
, irt_isnum(t
) ? CC_AE
: CC_NE
);
1458 if (LJ_64
&& irt_type(t
) >= IRT_NUM
) {
1459 lua_assert(irt_isinteger(t
) || irt_isnum(t
));
1460 emit_u32(as
, LJ_TISNUM
);
1461 emit_rmro(as
, XO_ARITHi
, XOg_CMP
, base
, ofs
+4);
1463 emit_i8(as
, irt_toitype(t
));
1464 emit_rmro(as
, XO_ARITHi8
, XOg_CMP
, base
, ofs
+4);
1469 /* -- Allocations --------------------------------------------------------- */
1472 static void asm_cnew(ASMState
*as
, IRIns
*ir
)
1474 CTState
*cts
= ctype_ctsG(J2G(as
->J
));
1475 CTypeID ctypeid
= (CTypeID
)IR(ir
->op1
)->i
;
1476 CTSize sz
= (ir
->o
== IR_CNEWI
|| ir
->op2
== REF_NIL
) ?
1477 lj_ctype_size(cts
, ctypeid
) : (CTSize
)IR(ir
->op2
)->i
;
1478 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_mem_newgco
];
1480 lua_assert(sz
!= CTSIZE_INVALID
);
1482 args
[0] = ASMREF_L
; /* lua_State *L */
1483 args
[1] = ASMREF_TMP1
; /* MSize size */
1485 asm_setupresult(as
, ir
, ci
); /* GCcdata * */
1487 /* Initialize immutable cdata object. */
1488 if (ir
->o
== IR_CNEWI
) {
1489 RegSet allow
= (RSET_GPR
& ~RSET_SCRATCH
);
1491 Reg r64
= sz
== 8 ? REX_64
: 0;
1492 if (irref_isk(ir
->op2
)) {
1493 IRIns
*irk
= IR(ir
->op2
);
1494 uint64_t k
= irk
->o
== IR_KINT64
? ir_k64(irk
)->u64
:
1495 (uint64_t)(uint32_t)irk
->i
;
1496 if (sz
== 4 || checki32((int64_t)k
)) {
1497 emit_i32(as
, (int32_t)k
);
1498 emit_rmro(as
, XO_MOVmi
, r64
, RID_RET
, sizeof(GCcdata
));
1500 emit_movtomro(as
, RID_ECX
+ r64
, RID_RET
, sizeof(GCcdata
));
1501 emit_loadu64(as
, RID_ECX
, k
);
1504 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1505 emit_movtomro(as
, r
+ r64
, RID_RET
, sizeof(GCcdata
));
1508 int32_t ofs
= sizeof(GCcdata
);
1511 lua_assert(ir
->o
== IR_HIOP
);
1514 if (irref_isk(ir
->op2
)) {
1515 emit_movmroi(as
, RID_RET
, ofs
, IR(ir
->op2
)->i
);
1517 Reg r
= ra_alloc1(as
, ir
->op2
, allow
);
1518 emit_movtomro(as
, r
, RID_RET
, ofs
);
1519 rset_clear(allow
, r
);
1521 if (ofs
== sizeof(GCcdata
)) break;
1525 lua_assert(sz
== 4 || sz
== 8);
1528 /* Combine initialization of marked, gct and ctypeid. */
1529 emit_movtomro(as
, RID_ECX
, RID_RET
, offsetof(GCcdata
, marked
));
1530 emit_gri(as
, XG_ARITHi(XOg_OR
), RID_ECX
,
1531 (int32_t)((~LJ_TCDATA
<<8)+(ctypeid
<<16)));
1532 emit_gri(as
, XG_ARITHi(XOg_AND
), RID_ECX
, LJ_GC_WHITES
);
1533 emit_opgl(as
, XO_MOVZXb
, RID_ECX
, gc
.currentwhite
);
1535 asm_gencall(as
, ci
, args
);
1536 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP1
), (int32_t)(sz
+sizeof(GCcdata
)));
1539 #define asm_cnew(as, ir) ((void)0)
1542 /* -- Write barriers ------------------------------------------------------ */
1544 static void asm_tbar(ASMState
*as
, IRIns
*ir
)
1546 Reg tab
= ra_alloc1(as
, ir
->op1
, RSET_GPR
);
1547 Reg tmp
= ra_scratch(as
, rset_exclude(RSET_GPR
, tab
));
1548 MCLabel l_end
= emit_label(as
);
1549 emit_movtomro(as
, tmp
, tab
, offsetof(GCtab
, gclist
));
1550 emit_setgl(as
, tab
, gc
.grayagain
);
1551 emit_getgl(as
, tmp
, gc
.grayagain
);
1552 emit_i8(as
, ~LJ_GC_BLACK
);
1553 emit_rmro(as
, XO_ARITHib
, XOg_AND
, tab
, offsetof(GCtab
, marked
));
1554 emit_sjcc(as
, CC_Z
, l_end
);
1555 emit_i8(as
, LJ_GC_BLACK
);
1556 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, tab
, offsetof(GCtab
, marked
));
1559 static void asm_obar(ASMState
*as
, IRIns
*ir
)
1561 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_barrieruv
];
1565 /* No need for other object barriers (yet). */
1566 lua_assert(IR(ir
->op1
)->o
== IR_UREFC
);
1567 ra_evictset(as
, RSET_SCRATCH
);
1568 l_end
= emit_label(as
);
1569 args
[0] = ASMREF_TMP1
; /* global_State *g */
1570 args
[1] = ir
->op1
; /* TValue *tv */
1571 asm_gencall(as
, ci
, args
);
1572 emit_loada(as
, ra_releasetmp(as
, ASMREF_TMP1
), J2G(as
->J
));
1573 obj
= IR(ir
->op1
)->r
;
1574 emit_sjcc(as
, CC_Z
, l_end
);
1575 emit_i8(as
, LJ_GC_WHITES
);
1576 if (irref_isk(ir
->op2
)) {
1577 GCobj
*vp
= ir_kgc(IR(ir
->op2
));
1578 emit_rma(as
, XO_GROUP3b
, XOg_TEST
, &vp
->gch
.marked
);
1580 Reg val
= ra_alloc1(as
, ir
->op2
, rset_exclude(RSET_SCRATCH
&RSET_GPR
, obj
));
1581 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, val
, (int32_t)offsetof(GChead
, marked
));
1583 emit_sjcc(as
, CC_Z
, l_end
);
1584 emit_i8(as
, LJ_GC_BLACK
);
1585 emit_rmro(as
, XO_GROUP3b
, XOg_TEST
, obj
,
1586 (int32_t)offsetof(GCupval
, marked
)-(int32_t)offsetof(GCupval
, tv
));
1589 /* -- FP/int arithmetic and logic operations ------------------------------ */
1591 /* Load reference onto x87 stack. Force a spill to memory if needed. */
1592 static void asm_x87load(ASMState
*as
, IRRef ref
)
1594 IRIns
*ir
= IR(ref
);
1595 if (ir
->o
== IR_KNUM
) {
1596 cTValue
*tv
= ir_knum(ir
);
1597 if (tvispzero(tv
)) /* Use fldz only for +0. */
1598 emit_x87op(as
, XI_FLDZ
);
1599 else if (tvispone(tv
))
1600 emit_x87op(as
, XI_FLD1
);
1602 emit_rma(as
, XO_FLDq
, XOg_FLDq
, tv
);
1603 } else if (ir
->o
== IR_CONV
&& ir
->op2
== IRCONV_NUM_INT
&& !ra_used(ir
) &&
1604 !irref_isk(ir
->op1
) && mayfuse(as
, ir
->op1
)) {
1605 IRIns
*iri
= IR(ir
->op1
);
1606 emit_rmro(as
, XO_FILDd
, XOg_FILDd
, RID_ESP
, ra_spill(as
, iri
));
1608 emit_mrm(as
, XO_FLDq
, XOg_FLDq
, asm_fuseload(as
, ref
, RSET_EMPTY
));
1612 /* Try to rejoin pow from EXP2, MUL and LOG2 (if still unsplit). */
1613 static int fpmjoin_pow(ASMState
*as
, IRIns
*ir
)
1615 IRIns
*irp
= IR(ir
->op1
);
1616 if (irp
== ir
-1 && irp
->o
== IR_MUL
&& !ra_used(irp
)) {
1617 IRIns
*irpp
= IR(irp
->op1
);
1618 if (irpp
== ir
-2 && irpp
->o
== IR_FPMATH
&&
1619 irpp
->op2
== IRFPM_LOG2
&& !ra_used(irpp
)) {
1620 /* The modified regs must match with the *.dasc implementation. */
1621 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM2
+1)|RID2RSET(RID_EAX
);
1623 if (ra_hasreg(ir
->r
))
1624 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1625 ra_evictset(as
, drop
);
1626 ra_destreg(as
, ir
, RID_XMM0
);
1627 emit_call(as
, lj_vm_pow_sse
);
1628 irx
= IR(irpp
->op1
);
1629 if (ra_noreg(irx
->r
) && ra_gethint(irx
->r
) == RID_XMM1
)
1630 irx
->r
= RID_INIT
; /* Avoid allocating xmm1 for x. */
1631 ra_left(as
, RID_XMM0
, irpp
->op1
);
1632 ra_left(as
, RID_XMM1
, irp
->op2
);
1639 static void asm_fpmath(ASMState
*as
, IRIns
*ir
)
1641 IRFPMathOp fpm
= ir
->o
== IR_FPMATH
? (IRFPMathOp
)ir
->op2
: IRFPM_OTHER
;
1642 if (fpm
== IRFPM_SQRT
) {
1643 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1644 Reg left
= asm_fuseload(as
, ir
->op1
, RSET_FPR
);
1645 emit_mrm(as
, XO_SQRTSD
, dest
, left
);
1646 } else if (fpm
<= IRFPM_TRUNC
) {
1647 if (as
->flags
& JIT_F_SSE4_1
) { /* SSE4.1 has a rounding instruction. */
1648 Reg dest
= ra_dest(as
, ir
, RSET_FPR
);
1649 Reg left
= asm_fuseload(as
, ir
->op1
, RSET_FPR
);
1650 /* ROUNDSD has a 4-byte opcode which doesn't fit in x86Op.
1651 ** Let's pretend it's a 3-byte opcode, and compensate afterwards.
1652 ** This is atrocious, but the alternatives are much worse.
1654 /* Round down/up/trunc == 1001/1010/1011. */
1655 emit_i8(as
, 0x09 + fpm
);
1656 emit_mrm(as
, XO_ROUNDSD
, dest
, left
);
1657 if (LJ_64
&& as
->mcp
[1] != (MCode
)(XO_ROUNDSD
>> 16)) {
1658 as
->mcp
[0] = as
->mcp
[1]; as
->mcp
[1] = 0x0f; /* Swap 0F and REX. */
1660 *--as
->mcp
= 0x66; /* 1st byte of ROUNDSD opcode. */
1661 } else { /* Call helper functions for SSE2 variant. */
1662 /* The modified regs must match with the *.dasc implementation. */
1663 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM3
+1)|RID2RSET(RID_EAX
);
1664 if (ra_hasreg(ir
->r
))
1665 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1666 ra_evictset(as
, drop
);
1667 ra_destreg(as
, ir
, RID_XMM0
);
1668 emit_call(as
, fpm
== IRFPM_FLOOR
? lj_vm_floor_sse
:
1669 fpm
== IRFPM_CEIL
? lj_vm_ceil_sse
: lj_vm_trunc_sse
);
1670 ra_left(as
, RID_XMM0
, ir
->op1
);
1672 } else if (fpm
== IRFPM_EXP2
&& fpmjoin_pow(as
, ir
)) {
1673 /* Rejoined to pow(). */
1674 } else { /* Handle x87 ops. */
1675 int32_t ofs
= sps_scale(ir
->s
); /* Use spill slot or temp slots. */
1677 if (ra_hasreg(dest
)) {
1679 ra_modified(as
, dest
);
1680 emit_rmro(as
, XMM_MOVRM(as
), dest
, RID_ESP
, ofs
);
1682 emit_rmro(as
, XO_FSTPq
, XOg_FSTPq
, RID_ESP
, ofs
);
1683 switch (fpm
) { /* st0 = lj_vm_*(st0) */
1684 case IRFPM_EXP
: emit_call(as
, lj_vm_exp_x87
); break;
1685 case IRFPM_EXP2
: emit_call(as
, lj_vm_exp2_x87
); break;
1686 case IRFPM_SIN
: emit_x87op(as
, XI_FSIN
); break;
1687 case IRFPM_COS
: emit_x87op(as
, XI_FCOS
); break;
1688 case IRFPM_TAN
: emit_x87op(as
, XI_FPOP
); emit_x87op(as
, XI_FPTAN
); break;
1689 case IRFPM_LOG
: case IRFPM_LOG2
: case IRFPM_LOG10
:
1690 /* Note: the use of fyl2xp1 would be pointless here. When computing
1691 ** log(1.0+eps) the precision is already lost after 1.0 is added.
1692 ** Subtracting 1.0 won't recover it. OTOH math.log1p would make sense.
1694 emit_x87op(as
, XI_FYL2X
); break;
1698 emit_x87op(as
, XI_FPATAN
); asm_x87load(as
, ir
->op2
); break;
1700 emit_x87op(as
, XI_FPOP1
); emit_x87op(as
, XI_FSCALE
); break;
1701 default: lua_assert(0); break;
1704 default: lua_assert(0); break;
1706 asm_x87load(as
, ir
->op1
);
1708 case IRFPM_LOG
: emit_x87op(as
, XI_FLDLN2
); break;
1709 case IRFPM_LOG2
: emit_x87op(as
, XI_FLD1
); break;
1710 case IRFPM_LOG10
: emit_x87op(as
, XI_FLDLG2
); break;
1712 if (ir
->o
== IR_LDEXP
) asm_x87load(as
, ir
->op2
);
1719 static void asm_fppowi(ASMState
*as
, IRIns
*ir
)
1721 /* The modified regs must match with the *.dasc implementation. */
1722 RegSet drop
= RSET_RANGE(RID_XMM0
, RID_XMM1
+1)|RID2RSET(RID_EAX
);
1723 if (ra_hasreg(ir
->r
))
1724 rset_clear(drop
, ir
->r
); /* Dest reg handled below. */
1725 ra_evictset(as
, drop
);
1726 ra_destreg(as
, ir
, RID_XMM0
);
1727 emit_call(as
, lj_vm_powi_sse
);
1728 ra_left(as
, RID_XMM0
, ir
->op1
);
1729 ra_left(as
, RID_EAX
, ir
->op2
);
1732 #if LJ_64 && LJ_HASFFI
1733 static void asm_arith64(ASMState
*as
, IRIns
*ir
, IRCallID id
)
1735 const CCallInfo
*ci
= &lj_ir_callinfo
[id
];
1739 asm_setupresult(as
, ir
, ci
);
1740 asm_gencall(as
, ci
, args
);
1744 static void asm_intmod(ASMState
*as
, IRIns
*ir
)
1746 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_vm_modi
];
1750 asm_setupresult(as
, ir
, ci
);
1751 asm_gencall(as
, ci
, args
);
1754 static int asm_swapops(ASMState
*as
, IRIns
*ir
)
1756 IRIns
*irl
= IR(ir
->op1
);
1757 IRIns
*irr
= IR(ir
->op2
);
1758 lua_assert(ra_noreg(irr
->r
));
1759 if (!irm_iscomm(lj_ir_mode
[ir
->o
]))
1760 return 0; /* Can't swap non-commutative operations. */
1761 if (irref_isk(ir
->op2
))
1762 return 0; /* Don't swap constants to the left. */
1763 if (ra_hasreg(irl
->r
))
1764 return 1; /* Swap if left already has a register. */
1765 if (ra_samehint(ir
->r
, irr
->r
))
1766 return 1; /* Swap if dest and right have matching hints. */
1767 if (as
->curins
> as
->loopref
) { /* In variant part? */
1768 if (ir
->op2
< as
->loopref
&& !irt_isphi(irr
->t
))
1769 return 0; /* Keep invariants on the right. */
1770 if (ir
->op1
< as
->loopref
&& !irt_isphi(irl
->t
))
1771 return 1; /* Swap invariants to the right. */
1773 if (opisfusableload(irl
->o
))
1774 return 1; /* Swap fusable loads to the right. */
1775 return 0; /* Otherwise don't swap. */
1778 static void asm_fparith(ASMState
*as
, IRIns
*ir
, x86Op xo
)
1780 IRRef lref
= ir
->op1
;
1781 IRRef rref
= ir
->op2
;
1782 RegSet allow
= RSET_FPR
;
1784 Reg right
= IR(rref
)->r
;
1785 if (ra_hasreg(right
)) {
1786 rset_clear(allow
, right
);
1787 ra_noweak(as
, right
);
1789 dest
= ra_dest(as
, ir
, allow
);
1792 } else if (ra_noreg(right
)) {
1793 if (asm_swapops(as
, ir
)) {
1794 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1796 right
= asm_fuseload(as
, rref
, rset_clear(allow
, dest
));
1798 emit_mrm(as
, xo
, dest
, right
);
1799 ra_left(as
, dest
, lref
);
1802 static void asm_intarith(ASMState
*as
, IRIns
*ir
, x86Arith xa
)
1804 IRRef lref
= ir
->op1
;
1805 IRRef rref
= ir
->op2
;
1806 RegSet allow
= RSET_GPR
;
1809 if (as
->flagmcp
== as
->mcp
) { /* Drop test r,r instruction. */
1811 as
->mcp
+= (LJ_64
&& *as
->mcp
!= XI_TEST
) ? 3 : 2;
1813 right
= IR(rref
)->r
;
1814 if (ra_hasreg(right
)) {
1815 rset_clear(allow
, right
);
1816 ra_noweak(as
, right
);
1818 dest
= ra_dest(as
, ir
, allow
);
1821 } else if (ra_noreg(right
) && !asm_isk32(as
, rref
, &k
)) {
1822 if (asm_swapops(as
, ir
)) {
1823 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
1825 right
= asm_fuseload(as
, rref
, rset_clear(allow
, dest
));
1827 if (irt_isguard(ir
->t
)) /* For IR_ADDOV etc. */
1828 asm_guardcc(as
, CC_O
);
1829 if (xa
!= XOg_X_IMUL
) {
1830 if (ra_hasreg(right
))
1831 emit_mrm(as
, XO_ARITH(xa
), REX_64IR(ir
, dest
), right
);
1833 emit_gri(as
, XG_ARITHi(xa
), REX_64IR(ir
, dest
), k
);
1834 } else if (ra_hasreg(right
)) { /* IMUL r, mrm. */
1835 emit_mrm(as
, XO_IMUL
, REX_64IR(ir
, dest
), right
);
1836 } else { /* IMUL r, r, k. */
1837 /* NYI: use lea/shl/add/sub (FOLD only does 2^k) depending on CPU. */
1838 Reg left
= asm_fuseload(as
, lref
, RSET_GPR
);
1840 if (checki8(k
)) { emit_i8(as
, k
); xo
= XO_IMULi8
;
1841 } else { emit_i32(as
, k
); xo
= XO_IMULi
; }
1842 emit_mrm(as
, xo
, REX_64IR(ir
, dest
), left
);
1845 ra_left(as
, dest
, lref
);
1848 /* LEA is really a 4-operand ADD with an independent destination register,
1849 ** up to two source registers and an immediate. One register can be scaled
1850 ** by 1, 2, 4 or 8. This can be used to avoid moves or to fuse several
1853 ** Currently only a few common cases are supported:
1854 ** - 3-operand ADD: y = a+b; y = a+k with a and b already allocated
1855 ** - Left ADD fusion: y = (a+b)+k; y = (a+k)+b
1856 ** - Right ADD fusion: y = a+(b+k)
1857 ** The ommited variants have already been reduced by FOLD.
1859 ** There are more fusion opportunities, like gathering shifts or joining
1860 ** common references. But these are probably not worth the trouble, since
1861 ** array indexing is not decomposed and already makes use of all fields
1862 ** of the ModRM operand.
1864 static int asm_lea(ASMState
*as
, IRIns
*ir
)
1866 IRIns
*irl
= IR(ir
->op1
);
1867 IRIns
*irr
= IR(ir
->op2
);
1868 RegSet allow
= RSET_GPR
;
1870 as
->mrm
.base
= as
->mrm
.idx
= RID_NONE
;
1871 as
->mrm
.scale
= XM_SCALE1
;
1873 if (ra_hasreg(irl
->r
)) {
1874 rset_clear(allow
, irl
->r
);
1875 ra_noweak(as
, irl
->r
);
1876 as
->mrm
.base
= irl
->r
;
1877 if (irref_isk(ir
->op2
) || ra_hasreg(irr
->r
)) {
1878 /* The PHI renaming logic does a better job in some cases. */
1879 if (ra_hasreg(ir
->r
) &&
1880 ((irt_isphi(irl
->t
) && as
->phireg
[ir
->r
] == ir
->op1
) ||
1881 (irt_isphi(irr
->t
) && as
->phireg
[ir
->r
] == ir
->op2
)))
1883 if (irref_isk(ir
->op2
)) {
1884 as
->mrm
.ofs
= irr
->i
;
1886 rset_clear(allow
, irr
->r
);
1887 ra_noweak(as
, irr
->r
);
1888 as
->mrm
.idx
= irr
->r
;
1890 } else if (irr
->o
== IR_ADD
&& mayfuse(as
, ir
->op2
) &&
1891 irref_isk(irr
->op2
)) {
1892 Reg idx
= ra_alloc1(as
, irr
->op1
, allow
);
1893 rset_clear(allow
, idx
);
1894 as
->mrm
.idx
= (uint8_t)idx
;
1895 as
->mrm
.ofs
= IR(irr
->op2
)->i
;
1899 } else if (ir
->op1
!= ir
->op2
&& irl
->o
== IR_ADD
&& mayfuse(as
, ir
->op1
) &&
1900 (irref_isk(ir
->op2
) || irref_isk(irl
->op2
))) {
1901 Reg idx
, base
= ra_alloc1(as
, irl
->op1
, allow
);
1902 rset_clear(allow
, base
);
1903 as
->mrm
.base
= (uint8_t)base
;
1904 if (irref_isk(ir
->op2
)) {
1905 as
->mrm
.ofs
= irr
->i
;
1906 idx
= ra_alloc1(as
, irl
->op2
, allow
);
1908 as
->mrm
.ofs
= IR(irl
->op2
)->i
;
1909 idx
= ra_alloc1(as
, ir
->op2
, allow
);
1911 rset_clear(allow
, idx
);
1912 as
->mrm
.idx
= (uint8_t)idx
;
1916 dest
= ra_dest(as
, ir
, allow
);
1917 emit_mrm(as
, XO_LEA
, dest
, RID_MRM
);
1918 return 1; /* Success. */
1921 static void asm_add(ASMState
*as
, IRIns
*ir
)
1923 if (irt_isnum(ir
->t
))
1924 asm_fparith(as
, ir
, XO_ADDSD
);
1925 else if ((as
->flags
& JIT_F_LEA_AGU
) || as
->flagmcp
== as
->mcp
||
1926 irt_is64(ir
->t
) || !asm_lea(as
, ir
))
1927 asm_intarith(as
, ir
, XOg_ADD
);
1930 static void asm_neg_not(ASMState
*as
, IRIns
*ir
, x86Group3 xg
)
1932 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1933 emit_rr(as
, XO_GROUP3
, REX_64IR(ir
, xg
), dest
);
1934 ra_left(as
, dest
, ir
->op1
);
1937 static void asm_min_max(ASMState
*as
, IRIns
*ir
, int cc
)
1939 Reg right
, dest
= ra_dest(as
, ir
, RSET_GPR
);
1940 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
1941 if (irref_isk(rref
)) { lref
= rref
; rref
= ir
->op1
; }
1942 right
= ra_alloc1(as
, rref
, rset_exclude(RSET_GPR
, dest
));
1943 emit_rr(as
, XO_CMOV
+ (cc
<<24), REX_64IR(ir
, dest
), right
);
1944 emit_rr(as
, XO_CMP
, REX_64IR(ir
, dest
), right
);
1945 ra_left(as
, dest
, lref
);
1948 static void asm_bitswap(ASMState
*as
, IRIns
*ir
)
1950 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
1951 as
->mcp
= emit_op(XO_BSWAP
+ ((dest
&7) << 24),
1952 REX_64IR(ir
, 0), dest
, 0, as
->mcp
, 1);
1953 ra_left(as
, dest
, ir
->op1
);
1956 static void asm_bitshift(ASMState
*as
, IRIns
*ir
, x86Shift xs
)
1958 IRRef rref
= ir
->op2
;
1959 IRIns
*irr
= IR(rref
);
1961 if (irref_isk(rref
)) { /* Constant shifts. */
1963 dest
= ra_dest(as
, ir
, RSET_GPR
);
1964 shift
= irr
->i
& (irt_is64(ir
->t
) ? 63 : 31);
1967 case 1: emit_rr(as
, XO_SHIFT1
, REX_64IR(ir
, xs
), dest
); break;
1968 default: emit_shifti(as
, REX_64IR(ir
, xs
), dest
, shift
); break;
1970 } else { /* Variable shifts implicitly use register cl (i.e. ecx). */
1972 dest
= ra_dest(as
, ir
, rset_exclude(RSET_GPR
, RID_ECX
));
1973 if (dest
== RID_ECX
) {
1974 dest
= ra_scratch(as
, rset_exclude(RSET_GPR
, RID_ECX
));
1975 emit_rr(as
, XO_MOV
, RID_ECX
, dest
);
1978 if (ra_noreg(right
))
1979 right
= ra_allocref(as
, rref
, RID2RSET(RID_ECX
));
1980 else if (right
!= RID_ECX
)
1981 ra_scratch(as
, RID2RSET(RID_ECX
));
1982 emit_rr(as
, XO_SHIFTcl
, REX_64IR(ir
, xs
), dest
);
1983 if (right
!= RID_ECX
) {
1984 ra_noweak(as
, right
);
1985 emit_rr(as
, XO_MOV
, RID_ECX
, right
);
1988 ra_left(as
, dest
, ir
->op1
);
1990 ** Note: avoid using the flags resulting from a shift or rotate!
1991 ** All of them cause a partial flag stall, except for r,1 shifts
1992 ** (but not rotates). And a shift count of 0 leaves the flags unmodified.
1996 /* -- Comparisons --------------------------------------------------------- */
1998 /* Virtual flags for unordered FP comparisons. */
1999 #define VCC_U 0x1000 /* Unordered. */
2000 #define VCC_P 0x2000 /* Needs extra CC_P branch. */
2001 #define VCC_S 0x4000 /* Swap avoids CC_P branch. */
2002 #define VCC_PS (VCC_P|VCC_S)
2004 /* Map of comparisons to flags. ORDER IR. */
2005 #define COMPFLAGS(ci, cin, cu, cf) ((ci)+((cu)<<4)+((cin)<<8)+(cf))
2006 static const uint16_t asm_compmap
[IR_ABC
+1] = {
2007 /* signed non-eq unsigned flags */
2008 /* LT */ COMPFLAGS(CC_GE
, CC_G
, CC_AE
, VCC_PS
),
2009 /* GE */ COMPFLAGS(CC_L
, CC_L
, CC_B
, 0),
2010 /* LE */ COMPFLAGS(CC_G
, CC_G
, CC_A
, VCC_PS
),
2011 /* GT */ COMPFLAGS(CC_LE
, CC_L
, CC_BE
, 0),
2012 /* ULT */ COMPFLAGS(CC_AE
, CC_A
, CC_AE
, VCC_U
),
2013 /* UGE */ COMPFLAGS(CC_B
, CC_B
, CC_B
, VCC_U
|VCC_PS
),
2014 /* ULE */ COMPFLAGS(CC_A
, CC_A
, CC_A
, VCC_U
),
2015 /* UGT */ COMPFLAGS(CC_BE
, CC_B
, CC_BE
, VCC_U
|VCC_PS
),
2016 /* EQ */ COMPFLAGS(CC_NE
, CC_NE
, CC_NE
, VCC_P
),
2017 /* NE */ COMPFLAGS(CC_E
, CC_E
, CC_E
, VCC_U
|VCC_P
),
2018 /* ABC */ COMPFLAGS(CC_BE
, CC_B
, CC_BE
, VCC_U
|VCC_PS
) /* Same as UGT. */
2021 /* FP and integer comparisons. */
2022 static void asm_comp(ASMState
*as
, IRIns
*ir
, uint32_t cc
)
2024 if (irt_isnum(ir
->t
)) {
2025 IRRef lref
= ir
->op1
;
2026 IRRef rref
= ir
->op2
;
2030 ** An extra CC_P branch is required to preserve ordered/unordered
2031 ** semantics for FP comparisons. This can be avoided by swapping
2032 ** the operands and inverting the condition (except for EQ and UNE).
2033 ** So always try to swap if possible.
2035 ** Another option would be to swap operands to achieve better memory
2036 ** operand fusion. But it's unlikely that this outweighs the cost
2037 ** of the extra branches.
2039 if (cc
& VCC_S
) { /* Swap? */
2040 IRRef tmp
= lref
; lref
= rref
; rref
= tmp
;
2041 cc
^= (VCC_PS
|(5<<4)); /* A <-> B, AE <-> BE, PS <-> none */
2043 left
= ra_alloc1(as
, lref
, RSET_FPR
);
2044 right
= asm_fuseload(as
, rref
, rset_exclude(RSET_FPR
, left
));
2045 l_around
= emit_label(as
);
2046 asm_guardcc(as
, cc
>> 4);
2047 if (cc
& VCC_P
) { /* Extra CC_P branch required? */
2048 if (!(cc
& VCC_U
)) {
2049 asm_guardcc(as
, CC_P
); /* Branch to exit for ordered comparisons. */
2050 } else if (l_around
!= as
->invmcp
) {
2051 emit_sjcc(as
, CC_P
, l_around
); /* Branch around for unordered. */
2053 /* Patched to mcloop by asm_loop_fixup. */
2056 emit_sjcc(as
, CC_P
, as
->mcp
);
2058 emit_jcc(as
, CC_P
, as
->mcp
);
2061 emit_mrm(as
, XO_UCOMISD
, left
, right
);
2063 IRRef lref
= ir
->op1
, rref
= ir
->op2
;
2064 IROp leftop
= (IROp
)(IR(lref
)->o
);
2065 Reg r64
= REX_64IR(ir
, 0);
2067 lua_assert(irt_is64(ir
->t
) || irt_isint(ir
->t
) ||
2068 irt_isu32(ir
->t
) || irt_isaddr(ir
->t
));
2069 /* Swap constants (only for ABC) and fusable loads to the right. */
2070 if (irref_isk(lref
) || (!irref_isk(rref
) && opisfusableload(leftop
))) {
2071 if ((cc
& 0xc) == 0xc) cc
^= 0x53; /* L <-> G, LE <-> GE */
2072 else if ((cc
& 0xa) == 0x2) cc
^= 0x55; /* A <-> B, AE <-> BE */
2073 lref
= ir
->op2
; rref
= ir
->op1
;
2075 if (asm_isk32(as
, rref
, &imm
)) {
2076 IRIns
*irl
= IR(lref
);
2077 /* Check wether we can use test ins. Not for unsigned, since CF=0. */
2078 int usetest
= (imm
== 0 && (cc
& 0xa) != 0x2);
2079 if (usetest
&& irl
->o
== IR_BAND
&& irl
+1 == ir
&& !ra_used(irl
)) {
2080 /* Combine comp(BAND(ref, r/imm), 0) into test mrm, r/imm. */
2081 Reg right
, left
= RID_NONE
;
2082 RegSet allow
= RSET_GPR
;
2083 if (!asm_isk32(as
, irl
->op2
, &imm
)) {
2084 left
= ra_alloc1(as
, irl
->op2
, allow
);
2085 rset_clear(allow
, left
);
2086 } else { /* Try to Fuse IRT_I8/IRT_U8 loads, too. See below. */
2087 IRIns
*irll
= IR(irl
->op1
);
2088 if (opisfusableload((IROp
)irll
->o
) &&
2089 (irt_isi8(irll
->t
) || irt_isu8(irll
->t
))) {
2090 IRType1 origt
= irll
->t
; /* Temporarily flip types. */
2091 irll
->t
.irt
= (irll
->t
.irt
& ~IRT_TYPE
) | IRT_INT
;
2092 as
->curins
--; /* Skip to BAND to avoid failing in noconflict(). */
2093 right
= asm_fuseload(as
, irl
->op1
, RSET_GPR
);
2096 if (right
!= RID_MRM
) goto test_nofuse
;
2097 /* Fusion succeeded, emit test byte mrm, imm8. */
2098 asm_guardcc(as
, cc
);
2099 emit_i8(as
, (imm
& 0xff));
2100 emit_mrm(as
, XO_GROUP3b
, XOg_TEST
, RID_MRM
);
2104 as
->curins
--; /* Skip to BAND to avoid failing in noconflict(). */
2105 right
= asm_fuseload(as
, irl
->op1
, allow
);
2106 as
->curins
++; /* Undo the above. */
2108 asm_guardcc(as
, cc
);
2109 if (ra_noreg(left
)) {
2111 emit_mrm(as
, XO_GROUP3
, r64
+ XOg_TEST
, right
);
2113 emit_mrm(as
, XO_TEST
, r64
+ left
, right
);
2117 if (opisfusableload((IROp
)irl
->o
) &&
2118 ((irt_isu8(irl
->t
) && checku8(imm
)) ||
2119 ((irt_isi8(irl
->t
) || irt_isi16(irl
->t
)) && checki8(imm
)) ||
2120 (irt_isu16(irl
->t
) && checku16(imm
) && checki8((int16_t)imm
)))) {
2121 /* Only the IRT_INT case is fused by asm_fuseload.
2122 ** The IRT_I8/IRT_U8 loads and some IRT_I16/IRT_U16 loads
2123 ** are handled here.
2124 ** Note that cmp word [mem], imm16 should not be generated,
2125 ** since it has a length-changing prefix. Compares of a word
2126 ** against a sign-extended imm8 are ok, however.
2128 IRType1 origt
= irl
->t
; /* Temporarily flip types. */
2129 irl
->t
.irt
= (irl
->t
.irt
& ~IRT_TYPE
) | IRT_INT
;
2130 left
= asm_fuseload(as
, lref
, RSET_GPR
);
2132 if (left
== RID_MRM
) { /* Fusion succeeded? */
2133 if (irt_isu8(irl
->t
) || irt_isu16(irl
->t
))
2134 cc
>>= 4; /* Need unsigned compare. */
2135 asm_guardcc(as
, cc
);
2137 emit_mrm(as
, (irt_isi8(origt
) || irt_isu8(origt
)) ?
2138 XO_ARITHib
: XO_ARITHiw8
, r64
+ XOg_CMP
, RID_MRM
);
2140 } /* Otherwise handle register case as usual. */
2142 left
= asm_fuseload(as
, lref
, RSET_GPR
);
2144 asm_guardcc(as
, cc
);
2145 if (usetest
&& left
!= RID_MRM
) {
2146 /* Use test r,r instead of cmp r,0. */
2147 emit_rr(as
, XO_TEST
, r64
+ left
, left
);
2148 if (irl
+1 == ir
) /* Referencing previous ins? */
2149 as
->flagmcp
= as
->mcp
; /* Set flag to drop test r,r if possible. */
2151 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), r64
+ left
, imm
);
2155 Reg left
= ra_alloc1(as
, lref
, RSET_GPR
);
2156 Reg right
= asm_fuseload(as
, rref
, rset_exclude(RSET_GPR
, left
));
2157 asm_guardcc(as
, cc
);
2158 emit_mrm(as
, XO_CMP
, r64
+ left
, right
);
2163 #if LJ_32 && LJ_HASFFI
2164 /* 64 bit integer comparisons in 32 bit mode. */
2165 static void asm_comp_int64(ASMState
*as
, IRIns
*ir
)
2167 uint32_t cc
= asm_compmap
[(ir
-1)->o
];
2168 RegSet allow
= RSET_GPR
;
2169 Reg lefthi
= RID_NONE
, leftlo
= RID_NONE
;
2170 Reg righthi
= RID_NONE
, rightlo
= RID_NONE
;
2174 as
->curins
--; /* Skip loword ins. Avoids failing in noconflict(), too. */
2176 /* Allocate/fuse hiword operands. */
2177 if (irref_isk(ir
->op2
)) {
2178 lefthi
= asm_fuseload(as
, ir
->op1
, allow
);
2180 lefthi
= ra_alloc1(as
, ir
->op1
, allow
);
2181 righthi
= asm_fuseload(as
, ir
->op2
, allow
);
2182 if (righthi
== RID_MRM
) {
2183 if (as
->mrm
.base
!= RID_NONE
) rset_clear(allow
, as
->mrm
.base
);
2184 if (as
->mrm
.idx
!= RID_NONE
) rset_clear(allow
, as
->mrm
.idx
);
2186 rset_clear(allow
, righthi
);
2189 mrm
= as
->mrm
; /* Save state for hiword instruction. */
2191 /* Allocate/fuse loword operands. */
2192 if (irref_isk((ir
-1)->op2
)) {
2193 leftlo
= asm_fuseload(as
, (ir
-1)->op1
, allow
);
2195 leftlo
= ra_alloc1(as
, (ir
-1)->op1
, allow
);
2196 rightlo
= asm_fuseload(as
, (ir
-1)->op2
, allow
);
2197 if (rightlo
== RID_MRM
) {
2198 if (as
->mrm
.base
!= RID_NONE
) rset_clear(allow
, as
->mrm
.base
);
2199 if (as
->mrm
.idx
!= RID_NONE
) rset_clear(allow
, as
->mrm
.idx
);
2201 rset_clear(allow
, rightlo
);
2205 /* All register allocations must be performed _before_ this point. */
2206 l_around
= emit_label(as
);
2207 as
->invmcp
= as
->flagmcp
= NULL
; /* Cannot use these optimizations. */
2209 /* Loword comparison and branch. */
2210 asm_guardcc(as
, cc
>> 4); /* Always use unsigned compare for loword. */
2211 if (ra_noreg(rightlo
)) {
2212 int32_t imm
= IR((ir
-1)->op2
)->i
;
2213 if (imm
== 0 && ((cc
>> 4) & 0xa) != 0x2 && leftlo
!= RID_MRM
)
2214 emit_rr(as
, XO_TEST
, leftlo
, leftlo
);
2216 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), leftlo
, imm
);
2218 emit_mrm(as
, XO_CMP
, leftlo
, rightlo
);
2221 /* Hiword comparison and branches. */
2222 if ((cc
& 15) != CC_NE
)
2223 emit_sjcc(as
, CC_NE
, l_around
); /* Hiword unequal: skip loword compare. */
2224 if ((cc
& 15) != CC_E
)
2225 asm_guardcc(as
, cc
>> 8); /* Hiword compare without equality check. */
2226 as
->mrm
= mrm
; /* Restore state. */
2227 if (ra_noreg(righthi
)) {
2228 int32_t imm
= IR(ir
->op2
)->i
;
2229 if (imm
== 0 && (cc
& 0xa) != 0x2 && lefthi
!= RID_MRM
)
2230 emit_rr(as
, XO_TEST
, lefthi
, lefthi
);
2232 emit_gmrmi(as
, XG_ARITHi(XOg_CMP
), lefthi
, imm
);
2234 emit_mrm(as
, XO_CMP
, lefthi
, righthi
);
2239 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
2241 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
2242 static void asm_hiop(ASMState
*as
, IRIns
*ir
)
2244 #if LJ_32 && LJ_HASFFI
2245 /* HIOP is marked as a store because it needs its own DCE logic. */
2246 int uselo
= ra_used(ir
-1), usehi
= ra_used(ir
); /* Loword/hiword used? */
2247 if (LJ_UNLIKELY(!(as
->flags
& JIT_F_OPT_DCE
))) uselo
= usehi
= 1;
2248 if ((ir
-1)->o
== IR_CONV
) { /* Conversions to/from 64 bit. */
2249 if (usehi
|| uselo
) {
2250 if (irt_isfp(ir
->t
))
2251 asm_conv_fp_int64(as
, ir
);
2253 asm_conv_int64_fp(as
, ir
);
2255 as
->curins
--; /* Always skip the CONV. */
2257 } else if ((ir
-1)->o
<= IR_NE
) { /* 64 bit integer comparisons. ORDER IR. */
2258 asm_comp_int64(as
, ir
);
2260 } else if ((ir
-1)->o
== IR_XSTORE
) {
2261 if ((ir
-1)->r
!= RID_SINK
)
2262 asm_fxstore(as
, ir
);
2265 if (!usehi
) return; /* Skip unused hiword op for all remaining ops. */
2266 switch ((ir
-1)->o
) {
2270 asm_intarith(as
, ir
, XOg_ADC
);
2271 asm_intarith(as
, ir
-1, XOg_ADD
);
2276 asm_intarith(as
, ir
, XOg_SBB
);
2277 asm_intarith(as
, ir
-1, XOg_SUB
);
2280 Reg dest
= ra_dest(as
, ir
, RSET_GPR
);
2281 emit_rr(as
, XO_GROUP3
, XOg_NEG
, dest
);
2283 emit_rr(as
, XO_ARITHi8
, XOg_ADC
, dest
);
2284 ra_left(as
, dest
, ir
->op1
);
2286 asm_neg_not(as
, ir
-1, XOg_NEG
);
2291 ra_destreg(as
, ir
, RID_RETHI
);
2293 ra_allocref(as
, ir
->op1
, RID2RSET(RID_RETLO
)); /* Mark call as used. */
2296 /* Nothing to do here. Handled by CNEWI itself. */
2298 default: lua_assert(0); break;
2301 UNUSED(as
); UNUSED(ir
); lua_assert(0); /* Unused on x64 or without FFI. */
2305 /* -- Stack handling ------------------------------------------------------ */
2307 /* Check Lua stack size for overflow. Use exit handler as fallback. */
2308 static void asm_stack_check(ASMState
*as
, BCReg topslot
,
2309 IRIns
*irp
, RegSet allow
, ExitNo exitno
)
2311 /* Try to get an unused temp. register, otherwise spill/restore eax. */
2312 Reg pbase
= irp
? irp
->r
: RID_BASE
;
2313 Reg r
= allow
? rset_pickbot(allow
) : RID_EAX
;
2314 emit_jcc(as
, CC_B
, exitstub_addr(as
->J
, exitno
));
2315 if (allow
== RSET_EMPTY
) /* Restore temp. register. */
2316 emit_rmro(as
, XO_MOV
, r
|REX_64
, RID_ESP
, 0);
2319 emit_gri(as
, XG_ARITHi(XOg_CMP
), r
, (int32_t)(8*topslot
));
2320 if (ra_hasreg(pbase
) && pbase
!= r
)
2321 emit_rr(as
, XO_ARITH(XOg_SUB
), r
, pbase
);
2323 emit_rmro(as
, XO_ARITH(XOg_SUB
), r
, RID_NONE
,
2324 ptr2addr(&J2G(as
->J
)->jit_base
));
2325 emit_rmro(as
, XO_MOV
, r
, r
, offsetof(lua_State
, maxstack
));
2326 emit_getgl(as
, r
, jit_L
);
2327 if (allow
== RSET_EMPTY
) /* Spill temp. register. */
2328 emit_rmro(as
, XO_MOVto
, r
|REX_64
, RID_ESP
, 0);
2331 /* Restore Lua stack from on-trace state. */
2332 static void asm_stack_restore(ASMState
*as
, SnapShot
*snap
)
2334 SnapEntry
*map
= &as
->T
->snapmap
[snap
->mapofs
];
2335 SnapEntry
*flinks
= &as
->T
->snapmap
[snap_nextofs(as
->T
, snap
)-1];
2336 MSize n
, nent
= snap
->nent
;
2337 /* Store the value of all modified slots to the Lua stack. */
2338 for (n
= 0; n
< nent
; n
++) {
2339 SnapEntry sn
= map
[n
];
2340 BCReg s
= snap_slot(sn
);
2341 int32_t ofs
= 8*((int32_t)s
-1);
2342 IRRef ref
= snap_ref(sn
);
2343 IRIns
*ir
= IR(ref
);
2344 if ((sn
& SNAP_NORESTORE
))
2346 if (irt_isnum(ir
->t
)) {
2347 Reg src
= ra_alloc1(as
, ref
, RSET_FPR
);
2348 emit_rmro(as
, XO_MOVSDto
, src
, RID_BASE
, ofs
);
2350 lua_assert(irt_ispri(ir
->t
) || irt_isaddr(ir
->t
) ||
2351 (LJ_DUALNUM
&& irt_isinteger(ir
->t
)));
2352 if (!irref_isk(ref
)) {
2353 Reg src
= ra_alloc1(as
, ref
, rset_exclude(RSET_GPR
, RID_BASE
));
2354 emit_movtomro(as
, REX_64IR(ir
, src
), RID_BASE
, ofs
);
2355 } else if (!irt_ispri(ir
->t
)) {
2356 emit_movmroi(as
, RID_BASE
, ofs
, ir
->i
);
2358 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
2359 if (s
!= 0) /* Do not overwrite link to previous frame. */
2360 emit_movmroi(as
, RID_BASE
, ofs
+4, (int32_t)(*flinks
--));
2362 if (!(LJ_64
&& irt_islightud(ir
->t
)))
2363 emit_movmroi(as
, RID_BASE
, ofs
+4, irt_toitype(ir
->t
));
2368 lua_assert(map
+ nent
== flinks
);
2371 /* -- GC handling --------------------------------------------------------- */
2373 /* Check GC threshold and do one or more GC steps. */
2374 static void asm_gc_check(ASMState
*as
)
2376 const CCallInfo
*ci
= &lj_ir_callinfo
[IRCALL_lj_gc_step_jit
];
2380 ra_evictset(as
, RSET_SCRATCH
);
2381 l_end
= emit_label(as
);
2382 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2383 asm_guardcc(as
, CC_NE
); /* Assumes asm_snap_prep() already done. */
2384 emit_rr(as
, XO_TEST
, RID_RET
, RID_RET
);
2385 args
[0] = ASMREF_TMP1
; /* global_State *g */
2386 args
[1] = ASMREF_TMP2
; /* MSize steps */
2387 asm_gencall(as
, ci
, args
);
2388 tmp
= ra_releasetmp(as
, ASMREF_TMP1
);
2389 emit_loada(as
, tmp
, J2G(as
->J
));
2390 emit_loadi(as
, ra_releasetmp(as
, ASMREF_TMP2
), as
->gcsteps
);
2391 /* Jump around GC step if GC total < GC threshold. */
2392 emit_sjcc(as
, CC_B
, l_end
);
2393 emit_opgl(as
, XO_ARITH(XOg_CMP
), tmp
, gc
.threshold
);
2394 emit_getgl(as
, tmp
, gc
.total
);
2399 /* -- Loop handling ------------------------------------------------------- */
2401 /* Fixup the loop branch. */
2402 static void asm_loop_fixup(ASMState
*as
)
2404 MCode
*p
= as
->mctop
;
2405 MCode
*target
= as
->mcp
;
2406 if (as
->realign
) { /* Realigned loops use short jumps. */
2407 as
->realign
= NULL
; /* Stop another retry. */
2408 lua_assert(((intptr_t)target
& 15) == 0);
2409 if (as
->loopinv
) { /* Inverted loop branch? */
2412 lua_assert(target
- p
>= -128);
2413 p
[-1] = (MCode
)(target
- p
); /* Patch sjcc. */
2414 if (as
->loopinv
== 2)
2415 p
[-3] = (MCode
)(target
- p
+ 2); /* Patch opt. short jp. */
2417 lua_assert(target
- p
>= -128);
2418 p
[-1] = (MCode
)(int8_t)(target
- p
); /* Patch short jmp. */
2424 if (as
->loopinv
) { /* Inverted loop branch? */
2425 /* asm_guardcc already inverted the jcc and patched the jmp. */
2428 *(int32_t *)(p
-4) = (int32_t)(target
- p
); /* Patch jcc. */
2429 if (as
->loopinv
== 2) {
2430 *(int32_t *)(p
-10) = (int32_t)(target
- p
+ 6); /* Patch opt. jp. */
2433 } else { /* Otherwise just patch jmp. */
2434 *(int32_t *)(p
-4) = (int32_t)(target
- p
);
2437 /* Realign small loops and shorten the loop branch. */
2438 if (newloop
>= p
- 128) {
2439 as
->realign
= newloop
; /* Force a retry and remember alignment. */
2440 as
->curins
= as
->stopins
; /* Abort asm_trace now. */
2441 as
->T
->nins
= as
->orignins
; /* Remove any added renames. */
2446 /* -- Head of trace ------------------------------------------------------- */
2448 /* Coalesce BASE register for a root trace. */
2449 static void asm_head_root_base(ASMState
*as
)
2451 IRIns
*ir
= IR(REF_BASE
);
2455 if (rset_test(as
->modset
, r
))
2456 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2458 emit_rr(as
, XO_MOV
, r
, RID_BASE
);
2462 /* Coalesce or reload BASE register for a side trace. */
2463 static RegSet
asm_head_side_base(ASMState
*as
, IRIns
*irp
, RegSet allow
)
2465 IRIns
*ir
= IR(REF_BASE
);
2469 if (rset_test(as
->modset
, r
))
2470 ir
->r
= RID_INIT
; /* No inheritance for modified BASE register. */
2472 rset_clear(allow
, r
); /* Mark same BASE register as coalesced. */
2473 } else if (ra_hasreg(irp
->r
) && rset_test(as
->freeset
, irp
->r
)) {
2474 rset_clear(allow
, irp
->r
);
2475 emit_rr(as
, XO_MOV
, r
, irp
->r
); /* Move from coalesced parent reg. */
2477 emit_getgl(as
, r
, jit_base
); /* Otherwise reload BASE. */
2483 /* -- Tail of trace ------------------------------------------------------- */
2485 /* Fixup the tail code. */
2486 static void asm_tail_fixup(ASMState
*as
, TraceNo lnk
)
2488 /* Note: don't use as->mcp swap + emit_*: emit_op overwrites more bytes. */
2489 MCode
*p
= as
->mctop
;
2491 int32_t spadj
= as
->T
->spadjust
;
2493 p
-= ((as
->flags
& JIT_F_LEA_AGU
) ? 7 : 6) + (LJ_64
? 1 : 0);
2496 /* Patch stack adjustment. */
2497 if (checki8(spadj
)) {
2503 *(int32_t *)p1
= spadj
;
2505 if ((as
->flags
& JIT_F_LEA_AGU
)) {
2509 p1
[-3] = (MCode
)XI_LEA
;
2510 p1
[-2] = MODRM(checki8(spadj
) ? XM_OFS8
: XM_OFS32
, RID_ESP
, RID_ESP
);
2511 p1
[-1] = MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
2516 p1
[-2] = (MCode
)(checki8(spadj
) ? XI_ARITHi8
: XI_ARITHi
);
2517 p1
[-1] = MODRM(XM_REG
, XOg_ADD
, RID_ESP
);
2520 /* Patch exit branch. */
2521 target
= lnk
? traceref(as
->J
, lnk
)->mcode
: (MCode
*)lj_vm_exit_interp
;
2522 *(int32_t *)(p
-4) = jmprel(p
, target
);
2524 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
2525 for (q
= as
->mctop
-1; q
>= p
; q
--)
2530 /* Prepare tail of code. */
2531 static void asm_tail_prep(ASMState
*as
)
2533 MCode
*p
= as
->mctop
;
2534 /* Realign and leave room for backwards loop branch or exit branch. */
2536 int i
= ((int)(intptr_t)as
->realign
) & 15;
2537 /* Fill unused mcode tail with NOPs to make the prefetcher happy. */
2541 p
-= (as
->loopinv
? 5 : 2); /* Space for short/near jmp. */
2543 p
-= 5; /* Space for exit branch (near jmp). */
2546 as
->invmcp
= as
->mcp
= p
;
2548 /* Leave room for ESP adjustment: add esp, imm or lea esp, [esp+imm] */
2549 as
->mcp
= p
- (((as
->flags
& JIT_F_LEA_AGU
) ? 7 : 6) + (LJ_64
? 1 : 0));
2554 /* -- Instruction dispatch ------------------------------------------------ */
2556 /* Assemble a single instruction. */
2557 static void asm_ir(ASMState
*as
, IRIns
*ir
)
2559 switch ((IROp
)ir
->o
) {
2560 /* Miscellaneous ops. */
2561 case IR_LOOP
: asm_loop(as
); break;
2562 case IR_NOP
: case IR_XBAR
: lua_assert(!ra_used(ir
)); break;
2564 ra_alloc1(as
, ir
->op1
, irt_isfp(ir
->t
) ? RSET_FPR
: RSET_GPR
); break;
2565 case IR_PHI
: asm_phi(as
, ir
); break;
2566 case IR_HIOP
: asm_hiop(as
, ir
); break;
2567 case IR_GCSTEP
: asm_gcstep(as
, ir
); break;
2569 /* Guarded assertions. */
2570 case IR_LT
: case IR_GE
: case IR_LE
: case IR_GT
:
2571 case IR_ULT
: case IR_UGE
: case IR_ULE
: case IR_UGT
:
2572 case IR_EQ
: case IR_NE
: case IR_ABC
:
2573 asm_comp(as
, ir
, asm_compmap
[ir
->o
]);
2576 case IR_RETF
: asm_retf(as
, ir
); break;
2579 case IR_BNOT
: asm_neg_not(as
, ir
, XOg_NOT
); break;
2580 case IR_BSWAP
: asm_bitswap(as
, ir
); break;
2582 case IR_BAND
: asm_intarith(as
, ir
, XOg_AND
); break;
2583 case IR_BOR
: asm_intarith(as
, ir
, XOg_OR
); break;
2584 case IR_BXOR
: asm_intarith(as
, ir
, XOg_XOR
); break;
2586 case IR_BSHL
: asm_bitshift(as
, ir
, XOg_SHL
); break;
2587 case IR_BSHR
: asm_bitshift(as
, ir
, XOg_SHR
); break;
2588 case IR_BSAR
: asm_bitshift(as
, ir
, XOg_SAR
); break;
2589 case IR_BROL
: asm_bitshift(as
, ir
, XOg_ROL
); break;
2590 case IR_BROR
: asm_bitshift(as
, ir
, XOg_ROR
); break;
2592 /* Arithmetic ops. */
2593 case IR_ADD
: asm_add(as
, ir
); break;
2595 if (irt_isnum(ir
->t
))
2596 asm_fparith(as
, ir
, XO_SUBSD
);
2597 else /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
2598 asm_intarith(as
, ir
, XOg_SUB
);
2601 if (irt_isnum(ir
->t
))
2602 asm_fparith(as
, ir
, XO_MULSD
);
2604 asm_intarith(as
, ir
, XOg_X_IMUL
);
2607 #if LJ_64 && LJ_HASFFI
2608 if (!irt_isnum(ir
->t
))
2609 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_divi64
:
2610 IRCALL_lj_carith_divu64
);
2613 asm_fparith(as
, ir
, XO_DIVSD
);
2616 #if LJ_64 && LJ_HASFFI
2617 if (!irt_isint(ir
->t
))
2618 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_modi64
:
2619 IRCALL_lj_carith_modu64
);
2626 if (irt_isnum(ir
->t
))
2627 asm_fparith(as
, ir
, XO_XORPS
);
2629 asm_neg_not(as
, ir
, XOg_NEG
);
2631 case IR_ABS
: asm_fparith(as
, ir
, XO_ANDPS
); break;
2634 if (irt_isnum(ir
->t
))
2635 asm_fparith(as
, ir
, XO_MINSD
);
2637 asm_min_max(as
, ir
, CC_G
);
2640 if (irt_isnum(ir
->t
))
2641 asm_fparith(as
, ir
, XO_MAXSD
);
2643 asm_min_max(as
, ir
, CC_L
);
2646 case IR_FPMATH
: case IR_ATAN2
: case IR_LDEXP
:
2650 #if LJ_64 && LJ_HASFFI
2651 if (!irt_isnum(ir
->t
))
2652 asm_arith64(as
, ir
, irt_isi64(ir
->t
) ? IRCALL_lj_carith_powi64
:
2653 IRCALL_lj_carith_powu64
);
2659 /* Overflow-checking arithmetic ops. Note: don't use LEA here! */
2660 case IR_ADDOV
: asm_intarith(as
, ir
, XOg_ADD
); break;
2661 case IR_SUBOV
: asm_intarith(as
, ir
, XOg_SUB
); break;
2662 case IR_MULOV
: asm_intarith(as
, ir
, XOg_X_IMUL
); break;
2664 /* Memory references. */
2665 case IR_AREF
: asm_aref(as
, ir
); break;
2666 case IR_HREF
: asm_href(as
, ir
); break;
2667 case IR_HREFK
: asm_hrefk(as
, ir
); break;
2668 case IR_NEWREF
: asm_newref(as
, ir
); break;
2669 case IR_UREFO
: case IR_UREFC
: asm_uref(as
, ir
); break;
2670 case IR_FREF
: asm_fref(as
, ir
); break;
2671 case IR_STRREF
: asm_strref(as
, ir
); break;
2673 /* Loads and stores. */
2674 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
2675 asm_ahuvload(as
, ir
);
2677 case IR_FLOAD
: case IR_XLOAD
: asm_fxload(as
, ir
); break;
2678 case IR_SLOAD
: asm_sload(as
, ir
); break;
2680 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: asm_ahustore(as
, ir
); break;
2681 case IR_FSTORE
: case IR_XSTORE
: asm_fxstore(as
, ir
); break;
2684 case IR_SNEW
: case IR_XSNEW
: asm_snew(as
, ir
); break;
2685 case IR_TNEW
: asm_tnew(as
, ir
); break;
2686 case IR_TDUP
: asm_tdup(as
, ir
); break;
2687 case IR_CNEW
: case IR_CNEWI
: asm_cnew(as
, ir
); break;
2689 /* Write barriers. */
2690 case IR_TBAR
: asm_tbar(as
, ir
); break;
2691 case IR_OBAR
: asm_obar(as
, ir
); break;
2693 /* Type conversions. */
2694 case IR_TOBIT
: asm_tobit(as
, ir
); break;
2695 case IR_CONV
: asm_conv(as
, ir
); break;
2696 case IR_TOSTR
: asm_tostr(as
, ir
); break;
2697 case IR_STRTO
: asm_strto(as
, ir
); break;
2700 case IR_CALLN
: case IR_CALLL
: case IR_CALLS
: asm_call(as
, ir
); break;
2701 case IR_CALLXS
: asm_callx(as
, ir
); break;
2702 case IR_CARG
: break;
2705 setintV(&as
->J
->errinfo
, ir
->o
);
2706 lj_trace_err_info(as
->J
, LJ_TRERR_NYIIR
);
2711 /* -- Trace setup --------------------------------------------------------- */
2713 /* Ensure there are enough stack slots for call arguments. */
2714 static Reg
asm_setup_call_slots(ASMState
*as
, IRIns
*ir
, const CCallInfo
*ci
)
2716 IRRef args
[CCI_NARGS_MAX
];
2718 asm_collectargs(as
, ir
, ci
, args
);
2719 nslots
= asm_count_call_slots(as
, ci
, args
);
2720 if (nslots
> as
->evenspill
) /* Leave room for args in stack slots. */
2721 as
->evenspill
= nslots
;
2723 return irt_isfp(ir
->t
) ? REGSP_HINT(RID_FPRET
) : REGSP_HINT(RID_RET
);
2725 return irt_isfp(ir
->t
) ? REGSP_INIT
: REGSP_HINT(RID_RET
);
2729 /* Target-specific setup. */
2730 static void asm_setup_target(ASMState
*as
)
2732 asm_exitstub_setup(as
, as
->T
->nsnap
);
2735 /* -- Trace patching ------------------------------------------------------ */
2737 /* Patch exit jumps of existing machine code to a new target. */
2738 void lj_asm_patchexit(jit_State
*J
, GCtrace
*T
, ExitNo exitno
, MCode
*target
)
2740 MCode
*p
= T
->mcode
;
2741 MCode
*mcarea
= lj_mcode_patch(J
, p
, 0);
2742 MSize len
= T
->szmcode
;
2743 MCode
*px
= exitstub_addr(J
, exitno
) - 6;
2744 MCode
*pe
= p
+len
-6;
2745 uint32_t stateaddr
= u32ptr(&J2G(J
)->vmstate
);
2746 if (len
> 5 && p
[len
-5] == XI_JMP
&& p
+len
-6 + *(int32_t *)(p
+len
-4) == px
)
2747 *(int32_t *)(p
+len
-4) = jmprel(p
+len
, target
);
2748 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
2750 if (*(uint32_t *)(p
+(LJ_64
? 3 : 2)) == stateaddr
&& p
[0] == XI_MOVmi
) {
2751 p
+= LJ_64
? 11 : 10;
2755 for (; p
< pe
; p
++) {
2756 if ((*(uint16_t *)p
& 0xf0ff) == 0x800f && p
+ *(int32_t *)(p
+2) == px
) {
2757 *(int32_t *)(p
+2) = jmprel(p
+6, target
);
2761 lj_mcode_sync(T
->mcode
, T
->mcode
+ T
->szmcode
);
2762 lj_mcode_patch(J
, mcarea
, 1);