2 ** x86/x64 instruction emitter.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
6 /* -- Emit basic instructions --------------------------------------------- */
8 #define MODRM(mode, r1, r2) ((MCode)((mode)+(((r1)&7)<<3)+((r2)&7)))
11 #define REXRB(p, rr, rb) \
12 { MCode rex = 0x40 + (((rr)>>1)&4) + (((rb)>>3)&1); \
13 if (rex != 0x40) *--(p) = rex; }
14 #define FORCE_REX 0x200
15 #define REX_64 (FORCE_REX|0x080000)
16 #define VEX_64 0x800000
18 #define REXRB(p, rr, rb) ((void)0)
24 #define REX_GC64 REX_64
29 #define emit_i8(as, i) (*--as->mcp = (MCode)(i))
30 #define emit_i32(as, i) (*(int32_t *)(as->mcp-4) = (i), as->mcp -= 4)
31 #define emit_u32(as, u) (*(uint32_t *)(as->mcp-4) = (u), as->mcp -= 4)
33 #define emit_x87op(as, xo) \
34 (*(uint16_t *)(as->mcp-2) = (uint16_t)(xo), as->mcp -= 2)
37 static LJ_AINLINE MCode
*emit_op(x86Op xo
, Reg rr
, Reg rb
, Reg rx
,
41 if (n
== -60) { /* VEX-encoded instruction */
43 xo
^= (((rr
>>1)&4)+((rx
>>2)&2)+((rb
>>3)&1))<<13;
45 *(uint32_t *)(p
+delta
-5) = (uint32_t)xo
;
48 #if defined(__GNUC__) || defined(__clang__)
49 if (__builtin_constant_p(xo
) && n
== -2)
50 p
[delta
-2] = (MCode
)(xo
>> 24);
51 else if (__builtin_constant_p(xo
) && n
== -3)
52 *(uint16_t *)(p
+delta
-3) = (uint16_t)(xo
>> 16);
55 *(uint32_t *)(p
+delta
-5) = (uint32_t)xo
;
59 uint32_t rex
= 0x40 + ((rr
>>1)&(4+(FORCE_REX
>>1)))+((rx
>>2)&2)+((rb
>>3)&1);
62 if (n
== -4) { *p
= (MCode
)rex
; rex
= (MCode
)(xo
>> 8); }
63 else if ((xo
& 0xffffff) == 0x6600fd) { *p
= (MCode
)rex
; rex
= 0x66; }
68 UNUSED(rr
); UNUSED(rb
); UNUSED(rx
);
74 #define emit_opm(xo, mode, rr, rb, p, delta) \
75 (p[(delta)-1] = MODRM((mode), (rr), (rb)), \
76 emit_op((xo), (rr), (rb), 0, (p), (delta)))
78 /* op + modrm + sib */
79 #define emit_opmx(xo, mode, scale, rr, rb, rx, p) \
80 (p[-1] = MODRM((scale), (rx), (rb)), \
81 p[-2] = MODRM((mode), (rr), RID_ESP), \
82 emit_op((xo), (rr), (rb), (rx), (p), -1))
85 static void emit_rr(ASMState
*as
, x86Op xo
, Reg r1
, Reg r2
)
88 as
->mcp
= emit_opm(xo
, XM_REG
, r1
, r2
, p
, 0);
91 #if LJ_64 && defined(LUA_USE_ASSERT)
92 /* [addr] is sign-extended in x64 and must be in lower 2G (not 4G). */
93 static int32_t ptr2addr(const void *p
)
95 lj_assertX((uintptr_t)p
< (uintptr_t)0x80000000, "pointer outside 2G range");
99 #define ptr2addr(p) (i32ptr((p)))
102 /* op r, [base+ofs] */
103 static void emit_rmro(ASMState
*as
, x86Op xo
, Reg rr
, Reg rb
, int32_t ofs
)
108 if (LJ_GC64
&& rb
== RID_RIP
) {
112 } else if (ofs
== 0 && (rb
&7) != RID_EBP
) {
114 } else if (checki8(ofs
)) {
122 if ((rb
&7) == RID_ESP
)
123 *--p
= MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
125 *(int32_t *)(p
-4) = ofs
;
127 p
[-5] = MODRM(XM_SCALE1
, RID_ESP
, RID_EBP
);
136 as
->mcp
= emit_opm(xo
, mode
, rr
, rb
, p
, 0);
139 /* op r, [base+idx*scale+ofs] */
140 static void emit_rmrxo(ASMState
*as
, x86Op xo
, Reg rr
, Reg rb
, Reg rx
,
141 x86Mode scale
, int32_t ofs
)
145 if (ofs
== 0 && (rb
&7) != RID_EBP
) {
147 } else if (checki8(ofs
)) {
155 as
->mcp
= emit_opmx(xo
, mode
, scale
, rr
, rb
, rx
, p
);
159 static void emit_gri(ASMState
*as
, x86Group xg
, Reg rb
, int32_t i
)
171 as
->mcp
= emit_opm(xo
, XM_REG
, (Reg
)(xg
& 7) | (rb
& REX_64
), rb
, p
, 0);
174 /* op [base+ofs], i */
175 static void emit_gmroi(ASMState
*as
, x86Group xg
, Reg rb
, int32_t ofs
,
186 emit_rmro(as
, xo
, (Reg
)(xg
& 7), rb
, ofs
);
189 #define emit_shifti(as, xg, r, i) \
190 (emit_i8(as, (i)), emit_rr(as, XO_SHIFTi, (Reg)(xg), (r)))
193 static void emit_mrm(ASMState
*as
, x86Op xo
, Reg rr
, Reg rb
)
196 x86Mode mode
= XM_REG
;
199 if (rb
== RID_NONE
) {
203 *(int32_t *)p
= as
->mrm
.ofs
;
204 if (as
->mrm
.idx
!= RID_NONE
)
207 *--p
= MODRM(XM_SCALE1
, RID_ESP
, RID_EBP
);
210 } else if (LJ_GC64
&& rb
== RID_RIP
) {
211 lj_assertA(as
->mrm
.idx
== RID_NONE
, "RIP-rel mrm cannot have index");
214 *(int32_t *)p
= as
->mrm
.ofs
;
216 if (as
->mrm
.ofs
== 0 && (rb
&7) != RID_EBP
) {
218 } else if (checki8(as
->mrm
.ofs
)) {
219 *--p
= (MCode
)as
->mrm
.ofs
;
223 *(int32_t *)p
= as
->mrm
.ofs
;
226 if (as
->mrm
.idx
!= RID_NONE
) {
228 as
->mcp
= emit_opmx(xo
, mode
, as
->mrm
.scale
, rr
, rb
, as
->mrm
.idx
, p
);
231 if ((rb
&7) == RID_ESP
)
232 *--p
= MODRM(XM_SCALE1
, RID_ESP
, RID_ESP
);
235 as
->mcp
= emit_opm(xo
, mode
, rr
, rb
, p
, 0);
239 static void emit_gmrmi(ASMState
*as
, x86Group xg
, Reg rb
, int32_t i
)
249 emit_mrm(as
, xo
, (Reg
)(xg
& 7) | (rb
& REX_64
), (rb
& ~REX_64
));
252 /* -- Emit loads/stores --------------------------------------------------- */
254 /* mov [base+ofs], i */
255 static void emit_movmroi(ASMState
*as
, Reg base
, int32_t ofs
, int32_t i
)
258 emit_rmro(as
, XO_MOVmi
, 0, base
, ofs
);
261 /* mov [base+ofs], r */
262 #define emit_movtomro(as, r, base, ofs) \
263 emit_rmro(as, XO_MOVto, (r), (base), (ofs))
265 /* Get/set global_State fields. */
266 #define emit_opgl(as, xo, r, field) \
267 emit_rma(as, (xo), (r), (void *)&J2G(as->J)->field)
268 #define emit_getgl(as, r, field) emit_opgl(as, XO_MOV, (r)|REX_GC64, field)
269 #define emit_setgl(as, r, field) emit_opgl(as, XO_MOVto, (r)|REX_GC64, field)
271 #define emit_setvmstate(as, i) \
272 (emit_i32(as, i), emit_opgl(as, XO_MOVmi, 0, vmstate))
274 /* mov r, i / xor r, r */
275 static void emit_loadi(ASMState
*as
, Reg r
, int32_t i
)
277 /* XOR r,r is shorter, but modifies the flags. This is bad for HIOP/jcc. */
278 if (i
== 0 && !(LJ_32
&& (IR(as
->curins
)->o
== IR_HIOP
||
279 (as
->curins
+1 < as
->T
->nins
&&
280 IR(as
->curins
+1)->o
== IR_HIOP
))) &&
281 !((*as
->mcp
== 0x0f && (as
->mcp
[1] & 0xf0) == XI_JCCn
) ||
282 (*as
->mcp
& 0xf0) == XI_JCCs
)) {
283 emit_rr(as
, XO_ARITH(XOg_XOR
), r
, r
);
286 *(int32_t *)(p
-4) = i
;
287 p
[-5] = (MCode
)(XI_MOVri
+(r
&7));
295 #define dispofs(as, k) \
296 ((intptr_t)((uintptr_t)(k) - (uintptr_t)J2GG(as->J)->dispatch))
297 #define mcpofs(as, k) \
298 ((intptr_t)((uintptr_t)(k) - (uintptr_t)as->mcp))
299 #define mctopofs(as, k) \
300 ((intptr_t)((uintptr_t)(k) - (uintptr_t)as->mctop))
302 #define emit_loada(as, r, addr) \
303 emit_loadu64(as, (r), (uintptr_t)(addr))
306 #define emit_loada(as, r, addr) \
307 emit_loadi(as, (r), ptr2addr((addr)))
311 /* mov r, imm64 or shorter 32 bit extended load. */
312 static void emit_loadu64(ASMState
*as
, Reg r
, uint64_t u64
)
314 if (checku32(u64
)) { /* 32 bit load clears upper 32 bits. */
315 emit_loadi(as
, r
, (int32_t)u64
);
316 } else if (checki32((int64_t)u64
)) { /* Sign-extended 32 bit load. */
318 *(int32_t *)(p
-4) = (int32_t)u64
;
319 as
->mcp
= emit_opm(XO_MOVmi
, XM_REG
, REX_64
, r
, p
, -4);
321 } else if (checki32(dispofs(as
, u64
))) {
322 emit_rmro(as
, XO_LEA
, r
|REX_64
, RID_DISPATCH
, (int32_t)dispofs(as
, u64
));
323 } else if (checki32(mcpofs(as
, u64
)) && checki32(mctopofs(as
, u64
))) {
324 /* Since as->realign assumes the code size doesn't change, check
325 ** RIP-relative addressing reachability for both as->mcp and as->mctop.
327 emit_rmro(as
, XO_LEA
, r
|REX_64
, RID_RIP
, (int32_t)mcpofs(as
, u64
));
329 } else { /* Full-size 64 bit load. */
331 *(uint64_t *)(p
-8) = u64
;
332 p
[-9] = (MCode
)(XI_MOVri
+(r
&7));
333 p
[-10] = 0x48 + ((r
>>3)&1);
341 static void emit_rma(ASMState
*as
, x86Op xo
, Reg rr
, const void *addr
)
344 if (checki32(dispofs(as
, addr
))) {
345 emit_rmro(as
, xo
, rr
, RID_DISPATCH
, (int32_t)dispofs(as
, addr
));
346 } else if (checki32(mcpofs(as
, addr
)) && checki32(mctopofs(as
, addr
))) {
347 emit_rmro(as
, xo
, rr
, RID_RIP
, (int32_t)mcpofs(as
, addr
));
348 } else if (!checki32((intptr_t)addr
)) {
351 /* We can't allocate a register here. Use and restore DISPATCH. Ugly. */
352 uint64_t dispaddr
= (uintptr_t)J2GG(as
->J
)->dispatch
;
353 uint8_t i8
= xo
== XO_GROUP3b
? *as
->mcp
++ : 0;
355 if (checku32(dispaddr
)) {
356 emit_loadi(as
, ra
, (int32_t)dispaddr
);
357 } else { /* Full-size 64 bit load. */
359 *(uint64_t *)(p
-8) = dispaddr
;
360 p
[-9] = (MCode
)(XI_MOVri
+(ra
&7));
361 p
[-10] = 0x48 + ((ra
>>3)&1);
365 if (xo
== XO_GROUP3b
) emit_i8(as
, i8
);
367 emit_rmro(as
, xo
, rr
, ra
, 0);
368 emit_loadu64(as
, ra
, (uintptr_t)addr
);
373 *(int32_t *)(p
-4) = ptr2addr(addr
);
375 p
[-5] = MODRM(XM_SCALE1
, RID_ESP
, RID_EBP
);
376 as
->mcp
= emit_opm(xo
, XM_OFS0
, rr
, RID_ESP
, p
, -5);
378 as
->mcp
= emit_opm(xo
, XM_OFS0
, rr
, RID_EBP
, p
, -4);
383 /* Load 64 bit IR constant into register. */
384 static void emit_loadk64(ASMState
*as
, Reg r
, IRIns
*ir
)
388 const uint64_t *k
= &ir_k64(ir
)->u64
;
389 if (rset_test(RSET_FPR
, r
)) {
397 emit_rr(as
, rset_test(RSET_FPR
, r
) ? XO_XORPS
: XO_ARITH(XOg_XOR
), r
, r
);
399 } else if (checki32((intptr_t)k
) || checki32(dispofs(as
, k
)) ||
400 (checki32(mcpofs(as
, k
)) && checki32(mctopofs(as
, k
)))) {
401 emit_rma(as
, xo
, r64
, k
);
404 lj_assertA(*k
== *(uint64_t*)(as
->mctop
- ir
->i
),
405 "bad interned 64 bit constant");
406 } else if (as
->curins
<= as
->stopins
&& rset_test(RSET_GPR
, r
)) {
407 emit_loadu64(as
, r
, *k
);
410 /* If all else fails, add the FP constant at the MCode area bottom. */
411 while ((uintptr_t)as
->mcbot
& 7) *as
->mcbot
++ = XI_INT3
;
412 *(uint64_t *)as
->mcbot
= *k
;
413 ir
->i
= (int32_t)(as
->mctop
- as
->mcbot
);
415 as
->mclim
= as
->mcbot
+ MCLIM_REDZONE
;
416 lj_mcode_commitbot(as
->J
, as
->mcbot
);
418 emit_rmro(as
, xo
, r64
, RID_RIP
, (int32_t)mcpofs(as
, as
->mctop
- ir
->i
));
421 emit_rma(as
, xo
, r64
, k
);
426 /* -- Emit control-flow instructions -------------------------------------- */
428 /* Label for short jumps. */
429 typedef MCode
*MCLabel
;
431 #if LJ_32 && LJ_HASFFI
432 /* jmp short target */
433 static void emit_sjmp(ASMState
*as
, MCLabel target
)
436 ptrdiff_t delta
= target
- p
;
437 lj_assertA(delta
== (int8_t)delta
, "short jump target out of range");
438 p
[-1] = (MCode
)(int8_t)delta
;
444 /* jcc short target */
445 static void emit_sjcc(ASMState
*as
, int cc
, MCLabel target
)
448 ptrdiff_t delta
= target
- p
;
449 lj_assertA(delta
== (int8_t)delta
, "short jump target out of range");
450 p
[-1] = (MCode
)(int8_t)delta
;
451 p
[-2] = (MCode
)(XI_JCCs
+(cc
&15));
455 /* jcc short (pending target) */
456 static MCLabel
emit_sjcc_label(ASMState
*as
, int cc
)
460 p
[-2] = (MCode
)(XI_JCCs
+(cc
&15));
465 /* Fixup jcc short target. */
466 static void emit_sfixup(ASMState
*as
, MCLabel source
)
468 source
[-1] = (MCode
)(as
->mcp
-source
);
471 /* Return label pointing to current PC. */
472 #define emit_label(as) ((as)->mcp)
474 /* Compute relative 32 bit offset for jump and call instructions. */
475 static LJ_AINLINE
int32_t jmprel(jit_State
*J
, MCode
*p
, MCode
*target
)
477 ptrdiff_t delta
= target
- p
;
479 lj_assertJ(delta
== (int32_t)delta
, "jump target out of range");
480 return (int32_t)delta
;
484 static void emit_jcc(ASMState
*as
, int cc
, MCode
*target
)
487 *(int32_t *)(p
-4) = jmprel(as
->J
, p
, target
);
488 p
[-5] = (MCode
)(XI_JCCn
+(cc
&15));
494 static void emit_jmp(ASMState
*as
, MCode
*target
)
497 *(int32_t *)(p
-4) = jmprel(as
->J
, p
, target
);
503 static void emit_call_(ASMState
*as
, MCode
*target
)
507 if (target
-p
!= (int32_t)(target
-p
)) {
508 /* Assumes RID_RET is never an argument to calls and always clobbered. */
509 emit_rr(as
, XO_GROUP5
, XOg_CALL
, RID_RET
);
510 emit_loadu64(as
, RID_RET
, (uint64_t)target
);
514 *(int32_t *)(p
-4) = jmprel(as
->J
, p
, target
);
519 #define emit_call(as, f) emit_call_(as, (MCode *)(void *)(f))
521 /* -- Emit generic operations --------------------------------------------- */
523 /* Use 64 bit operations to handle 64 bit IR types. */
525 #define REX_64IR(ir, r) ((r) + (irt_is64((ir)->t) ? REX_64 : 0))
526 #define VEX_64IR(ir, r) ((r) + (irt_is64((ir)->t) ? VEX_64 : 0))
528 #define REX_64IR(ir, r) (r)
529 #define VEX_64IR(ir, r) (r)
532 /* Generic move between two regs. */
533 static void emit_movrr(ASMState
*as
, IRIns
*ir
, Reg dst
, Reg src
)
536 if (dst
< RID_MAX_GPR
)
537 emit_rr(as
, XO_MOV
, REX_64IR(ir
, dst
), src
);
539 emit_rr(as
, XO_MOVAPS
, dst
, src
);
542 /* Generic load of register with base and (small) offset address. */
543 static void emit_loadofs(ASMState
*as
, IRIns
*ir
, Reg r
, Reg base
, int32_t ofs
)
546 emit_rmro(as
, XO_MOV
, REX_64IR(ir
, r
), base
, ofs
);
548 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_MOVSD
: XO_MOVSS
, r
, base
, ofs
);
551 /* Generic store of register with base and (small) offset address. */
552 static void emit_storeofs(ASMState
*as
, IRIns
*ir
, Reg r
, Reg base
, int32_t ofs
)
555 emit_rmro(as
, XO_MOVto
, REX_64IR(ir
, r
), base
, ofs
);
557 emit_rmro(as
, irt_isnum(ir
->t
) ? XO_MOVSDto
: XO_MOVSSto
, r
, base
, ofs
);
560 /* Add offset to pointer. */
561 static void emit_addptr(ASMState
*as
, Reg r
, int32_t ofs
)
564 emit_gri(as
, XG_ARITHi(XOg_ADD
), r
|REX_GC64
, ofs
);
568 #define emit_spsub(as, ofs) emit_addptr(as, RID_ESP|REX_64, -(ofs))
570 /* Prefer rematerialization of BASE/L from global_State over spills. */
571 #define emit_canremat(ref) ((ref) <= REF_BASE)