?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
declare_folded_class NO LONGER _in_file
[hiphop-php.git] / hphp / util / asm-x64-intelxed.h
blob7ec4e7b46168ec2d4f7ce48e6394a02ae877f021
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 | Copyright (c) 2018 Intel Corporation |
7 +----------------------------------------------------------------------+
8 | This source file is subject to version 3.01 of the PHP license, |
9 | that is bundled with this package in the file LICENSE, and is |
10 | available through the world-wide-web at the following url: |
11 | http://www.php.net/license/3_01.txt |
12 | If you did not receive a copy of the PHP license and are unable to |
13 | obtain it through the world-wide-web, please send a note to |
14 | license@php.net so we can mail you a copy immediately. |
15 +----------------------------------------------------------------------+
16 */
17 #pragma once
18
19 extern "C" {
20 #include <xed-interface.h>
21 }
22
23 #include <tbb/concurrent_unordered_map.h>
24
25 /*
26 * A macro assembler for x64, based on the Intel XED library, that strives
27 * for low coupling to the runtime environment and ease of extendability.
28 */
29 namespace HPHP::jit {
30
31 struct XedInit {
32 XedInit() {
33 xed_tables_init();
34 }
35 };
36
37
38 ///////////////////////////////////////////////////////////////////////////////
39
40 struct XedAssembler final : public X64AssemblerBase {
41 private:
42 static constexpr xed_state_t kXedState = {
43 XED_MACHINE_MODE_LONG_64,
44 XED_ADDRESS_WIDTH_64b
45 };
46
47 CodeAddress dest() const {
48 codeBlock.assertCanEmit(XED_MAX_INSTRUCTION_BYTES);
49 return codeBlock.toDestAddress(codeBlock.frontier());
50 }
51
52 static constexpr auto nullrip = RIPRelativeRef(DispRIP(0));
53
54 public:
55 explicit XedAssembler(CodeBlock& cb) : X64AssemblerBase(cb) {}
56
57 XedAssembler(const XedAssembler&) = delete;
58 XedAssembler& operator=(const XedAssembler&) = delete;
59 /*
60 * The following section defines the main interface for emitting
61 * x64.
62 *
63 * Simple Examples:
64 *
65 * a. movq (rax, rbx); // order is AT&T: src, dest
66 * a. loadq (*rax, rbx); // loads from *rax
67 * a. loadq (rax[0], rbx); // also loads from *rax
68 * a. storeq (rcx, rax[0xc]); // store to rax + 0xc
69 * a. addq (0x1, rbx); // increment rbx
70 *
71 * Addressing with index registers:
72 *
73 * a. movl (index, ecx);
74 * a. loadq (*rax, rbx);
75 * a. storeq (rbx, rbx[rcx*8]);
76 * a. call (rax); // indirect call
77 *
78 */
79
80 #define BYTE_LOAD_OP(name, instr) \
81 void name##b(MemoryRef m, Reg8 r) { xedInstrMR(instr, m, r); }
82
83 #define LOAD_OP(name, instr) \
84 void name##q(MemoryRef m, Reg64 r) { xedInstrMR(instr, m, r); } \
85 void name##l(MemoryRef m, Reg32 r) { xedInstrMR(instr, m, r); } \
86 void name##w(MemoryRef m, Reg16 r) { xedInstrMR(instr, m, r); } \
87 void name##q(RIPRelativeRef m, Reg64 r) { xedInstrMR(instr, m, r); }\
88 BYTE_LOAD_OP(name, instr)
89
90 #define BYTE_STORE_OP(name, instr) \
91 void name##b(Reg8 r, MemoryRef m) { xedInstrRM(instr, r, m); } \
92 void name##b(Immed i, MemoryRef m) { xedInstrIM(instr, i, m, \
93 sz::byte); }
94
95 #define STORE_OP(name, instr) \
96 void name##w(Immed i, MemoryRef m) { \
97 xedInstrIM(instr, i, m, IMMPROP(sz::word, \
98 sz::word | sz::byte), sz::word); \
99 } \
100 void name##l(Immed i, MemoryRef m) { \
101 xedInstrIM(instr, i, m, IMMPROP(sz::dword, \
102 sz::dword | sz::byte), sz::dword);\
103 } \
104 void name##w(Reg16 r, MemoryRef m) { xedInstrRM(instr, r, m); } \
105 void name##l(Reg32 r, MemoryRef m) { xedInstrRM(instr, r, m); } \
106 void name##q(Reg64 r, MemoryRef m) { xedInstrRM(instr, r, m); } \
107 BYTE_STORE_OP(name, instr)
108
109 #define BYTE_REG_OP(name, instr) \
110 void name##b(Reg8 r1, Reg8 r2) { xedInstrRR(instr, r1, r2);} \
111 void name##b(Immed i, Reg8 r) { xedInstrIR(instr, i, r); }
112
113 #define REG_OP(name, instr) \
114 void name##q(Reg64 r1, Reg64 r2) { xedInstrRR(instr, r1, r2); } \
115 void name##l(Reg32 r1, Reg32 r2) { xedInstrRR(instr, r1, r2); } \
116 void name##w(Reg16 r1, Reg16 r2) { xedInstrRR(instr, r1, r2); } \
117 void name##l(Immed i, Reg32 r) { \
118 xedInstrIR(instr, i, r, IMMPROP(sz::dword, \
119 sz::dword | sz::byte)); \
120 } \
121 void name##w(Immed i, Reg16 r) { \
122 xedInstrIR(instr, i, r, IMMPROP(sz::word, \
123 sz::word | sz::byte)); \
124 } \
125 BYTE_REG_OP(name, instr)
126
127 #define IMM64_STORE_OP(name, instr) \
128 void name##q(Immed i, MemoryRef m) { \
129 xedInstrIM(instr, i, m, IMMPROP(sz::dword, \
130 sz::dword | sz::byte), sz::qword);\
131 }
132
133 #define IMM64R_OP(name, instr) \
134 void name##q(Immed imm, Reg64 r) { \
135 always_assert(imm.fits(sz::dword)); \
136 xedInstrIR(instr, imm, r, IMMPROP(sz::dword, \
137 sz::dword | sz::byte)); \
138 }
139
140 #define FULL_OP(name, instr) \
141 LOAD_OP(name, instr) \
142 STORE_OP(name, instr) \
143 REG_OP(name, instr) \
144 IMM64_STORE_OP(name, instr) \
145 IMM64R_OP(name, instr)
146
147 // We rename x64's mov to store and load for improved code
148 // readability.
149 #define IMMPROP(size, allsizes) size
150 LOAD_OP (load, XED_ICLASS_MOV)
151 STORE_OP (store,XED_ICLASS_MOV)
152 IMM64_STORE_OP (store,XED_ICLASS_MOV)
153 REG_OP (mov, XED_ICLASS_MOV)
154 FULL_OP (test, XED_ICLASS_TEST)
155 #undef IMMPROP
156
157 #define IMMPROP(size, allsizes) allsizes
158 FULL_OP(add, XED_ICLASS_ADD)
159 FULL_OP(xor, XED_ICLASS_XOR)
160 FULL_OP(sub, XED_ICLASS_SUB)
161 FULL_OP(and, XED_ICLASS_AND)
162 FULL_OP(or, XED_ICLASS_OR)
163 FULL_OP(cmp, XED_ICLASS_CMP)
164 FULL_OP(sbb, XED_ICLASS_SBB)
165 #undef IMMPROP
166
167 #undef IMM64R_OP
168 #undef FULL_OP
169 #undef REG_OP
170 #undef STORE_OP
171 #undef LOAD_OP
172 #undef BYTE_LOAD_OP
173 #undef BYTE_STORE_OP
174 #undef BYTE_REG_OP
175 #undef IMM64_STORE_OP
176
177 // 64-bit immediates work with mov to a register.
178 void movq(Immed64 imm, Reg64 r) { xedInstrIR(XED_ICLASS_MOV, imm, r,
179 sz::qword | sz::dword); }
180
181 // movzbx is a special snowflake. We don't have movzbq because it behaves
182 // exactly the same as movzbl but takes an extra byte.
183 void loadzbl(MemoryRef m, Reg32 r) { xedInstrMR(XED_ICLASS_MOVZX,
184 m, r, sz::byte); }
185 void loadzwl(MemoryRef m, Reg32 r) { xedInstrMR(XED_ICLASS_MOVZX,
186 m, r, sz::word); }
187 void movzbl(Reg8 src, Reg32 dest) { xedInstrRR(XED_ICLASS_MOVZX,
188 src, dest); }
189 void movsbl(Reg8 src, Reg32 dest) { xedInstrRR(XED_ICLASS_MOVSX,
190 src, dest); }
191 void movzwl(Reg16 src, Reg32 dest) { xedInstrRR(XED_ICLASS_MOVZX,
192 src, dest); }
193
194 void loadsbq(MemoryRef m, Reg64 r) { xedInstrMR(XED_ICLASS_MOVSX,
195 m, r, sz::byte); }
196 void movsbq(Reg8 src, Reg64 dest) { xedInstrRR(XED_ICLASS_MOVSX,
197 src, dest); }
198 void crc32q(Reg64 src, Reg64 dest) { xedInstrRR(XED_ICLASS_CRC32,
199 src, dest); }
200
201 void lea(MemoryRef p, Reg64 reg) { xedInstrMR(XED_ICLASS_LEA, p, reg); }
202 void lea(RIPRelativeRef p, Reg64 reg) { xedInstrMR(XED_ICLASS_LEA, p, reg); }
203
204 void xchgq(Reg64 r1, Reg64 r2) { xedInstrRR(XED_ICLASS_XCHG, r1, r2); }
205 void xchgl(Reg32 r1, Reg32 r2) { xedInstrRR(XED_ICLASS_XCHG, r1, r2); }
206 void xchgb(Reg8 r1, Reg8 r2) { xedInstrRR(XED_ICLASS_XCHG, r1, r2); }
207
208 void imul(Reg64 r1, Reg64 r2) { xedInstrRR(XED_ICLASS_IMUL, r1, r2); }
209
210 void push(Reg64 r) { xedInstrR(XED_ICLASS_PUSH, r); }
211 void pushl(Reg32 r) { xedInstrR(XED_ICLASS_PUSH, r); }
212 void pop (Reg64 r) { xedInstrR(XED_ICLASS_POP, r); }
213 void idiv(Reg64 r) { xedInstrR(XED_ICLASS_IDIV, r); }
214 void incq(Reg64 r) { xedInstrR(XED_ICLASS_INC, r); }
215 void incl(Reg32 r) { xedInstrR(XED_ICLASS_INC, r); }
216 void incw(Reg16 r) { xedInstrR(XED_ICLASS_INC, r); }
217 void decq(Reg64 r) { xedInstrR(XED_ICLASS_DEC, r); }
218 void decl(Reg32 r) { xedInstrR(XED_ICLASS_DEC, r); }
219 void decw(Reg16 r) { xedInstrR(XED_ICLASS_DEC, r); }
220 void notb(Reg8 r) { xedInstrR(XED_ICLASS_NOT, r); }
221 void not(Reg64 r) { xedInstrR(XED_ICLASS_NOT, r); }
222 void neg(Reg64 r) { xedInstrR(XED_ICLASS_NEG, r); }
223 void negb(Reg8 r) { xedInstrR(XED_ICLASS_NEG, r); }
224 void ret() { xedInstr(XED_ICLASS_RET_NEAR); }
225 void ret(Immed i) { xedInstrI(XED_ICLASS_IRET, i,
226 sz::word); }
227 void cqo() { xedInstr(XED_ICLASS_CQO); }
228 void nop() { xedInstr(XED_ICLASS_NOP, sz::byte); }
229 void int3() { xedInstr(XED_ICLASS_INT3, sz::byte); }
230 void ud2() { xedInstr(XED_ICLASS_UD2, sz::byte); }
231 void pushf() { xedInstr(XED_ICLASS_PUSHF, sz::word); }
232 void popf() { xedInstr(XED_ICLASS_POPF, sz::word); }
233 void lock() { always_assert(false); }
234
235 void push(MemoryRef m) { xedInstrM(XED_ICLASS_PUSH, m); }
236 void pop (MemoryRef m) { xedInstrM(XED_ICLASS_POP, m); }
237 void prefetch(MemoryRef m) { xedInstrM(XED_ICLASS_PREFETCHT1, m); }
238 void incq(MemoryRef m) { xedInstrM(XED_ICLASS_INC, m); }
239 void incl(MemoryRef m) { xedInstrM(XED_ICLASS_INC, m, sz::dword); }
240 void incw(MemoryRef m) { xedInstrM(XED_ICLASS_INC, m, sz::word); }
241 void decqlock(MemoryRef m) { xedInstrM(XED_ICLASS_DEC_LOCK, m); }
242 void decq(MemoryRef m) { xedInstrM(XED_ICLASS_DEC, m); }
243 void decl(MemoryRef m) { xedInstrM(XED_ICLASS_DEC, m, sz::dword); }
244 void decw(MemoryRef m) { xedInstrM(XED_ICLASS_DEC, m, sz::word); }
245
246 //special case for push(imm)
247 void push(Immed64 i) {
248 xed_encoder_operand_t op = toXedOperand(i, sz::byte | sz::word | sz::dword);
249 xedEmit(XED_ICLASS_PUSH, op, op.width_bits < 32 ? 16 : 64);
250 }
251
252 void movups(RegXMM x, MemoryRef m) { xedInstrRM(XED_ICLASS_MOVUPS,
253 x, m, sz::qword * 2); }
254 void movups(MemoryRef m, RegXMM x) { xedInstrMR(XED_ICLASS_MOVUPS,
255 m, x, sz::qword * 2); }
256 void movdqu(RegXMM x, MemoryRef m) { xedInstrRM(XED_ICLASS_MOVDQU,
257 x, m); }
258 void movdqu(MemoryRef m, RegXMM x) { xedInstrMR(XED_ICLASS_MOVDQU,
259 m, x); }
260 void movdqa(RegXMM x, RegXMM y) { xedInstrRR(XED_ICLASS_MOVDQA,
261 y, x); }
262 void movdqa(RegXMM x, MemoryRef m) { xedInstrRM(XED_ICLASS_MOVDQA,
263 x, m); }
264 void movdqa(MemoryRef m, RegXMM x) { xedInstrMR(XED_ICLASS_MOVDQA,
265 m, x); }
266 void movsd (RegXMM x, RegXMM y) { xedInstrRR(XED_ICLASS_MOVSD_XMM,
267 y, x); }
268 void movsd (RegXMM x, MemoryRef m) { xedInstrRM(XED_ICLASS_MOVSD_XMM,
269 x, m); }
270 void movsd (MemoryRef m, RegXMM x) { xedInstrMR(XED_ICLASS_MOVSD_XMM,
271 m, x); }
272 void movsd (RIPRelativeRef m, RegXMM x) { xedInstrMR(XED_ICLASS_MOVSD_XMM,
273 m, x); }
274 void lddqu (MemoryRef m, RegXMM x) { xedInstrMR(XED_ICLASS_LDDQU,
275 m, x); }
276 void unpcklpd(RegXMM s, RegXMM d) { xedInstrRR(XED_ICLASS_UNPCKLPD,
277 d, s); }
278
279 void rorq (Immed i, Reg64 r) { xedInstrIR(XED_ICLASS_ROR, i, r, sz::byte); }
280 void shlq (Immed i, Reg64 r) { xedInstrIR(XED_ICLASS_SHL, i, r, sz::byte); }
281 void shrq (Immed i, Reg64 r) { xedInstrIR(XED_ICLASS_SHR, i, r, sz::byte); }
282 void sarq (Immed i, Reg64 r) { xedInstrIR(XED_ICLASS_SAR, i, r, sz::byte); }
283 void shll (Immed i, Reg32 r) { xedInstrIR(XED_ICLASS_SHL, i, r, sz::byte); }
284 void shrl (Immed i, Reg32 r) { xedInstrIR(XED_ICLASS_SHR, i, r, sz::byte); }
285 void shlw (Immed i, Reg16 r) { xedInstrIR(XED_ICLASS_SHL, i, r, sz::byte); }
286 void shrw (Immed i, Reg16 r) { xedInstrIR(XED_ICLASS_SHR, i, r, sz::byte); }
287
288 void shlq (Reg64 r) { xedInstrRR_CL(XED_ICLASS_SHL, r); }
289 void shrq (Reg64 r) { xedInstrRR_CL(XED_ICLASS_SHR, r); }
290 void sarq (Reg64 r) { xedInstrRR_CL(XED_ICLASS_SAR, r); }
291
292 void btrq (Immed i, Reg64 r) { xedInstrIR(XED_ICLASS_BTR, i, r, sz::byte); }
293
294 void roundsd (RoundDirection d, RegXMM src, RegXMM dst) {
295 Immed i((int)d);
296 xedInstrIRR(XED_ICLASS_ROUNDSD, dst, src, i, sz::byte);
297 }
298
299 void cmpsd(RegXMM src, RegXMM dst, ComparisonPred pred) {
300 Immed i((int)pred);
301 xedInstrIRR(XED_ICLASS_CMPSD_XMM, dst, src, i, sz::byte);
302 }
303
304 /*
305 * Control-flow directives. Primitive labeling/patching facilities
306 * are available, as well as slightly higher-level ones via the
307 * Label class.
308 */
309
310 void jmp(Reg64 r) { xedInstrR(XED_ICLASS_JMP, r); }
311 void jmp(MemoryRef m) { xedInstrM(XED_ICLASS_JMP, m); }
312 void jmp(RIPRelativeRef m) { xedInstrM(XED_ICLASS_JMP, m); }
313 void call(Reg64 r) { xedInstrR(XED_ICLASS_CALL_NEAR, r); }
314 void call(MemoryRef m) { xedInstrM(XED_ICLASS_CALL_NEAR, m); }
315 void call(RIPRelativeRef m) { xedInstrM(XED_ICLASS_CALL_NEAR, m); }
316
317 void jmp8(CodeAddress dest) { xedInstrRelBr(XED_ICLASS_JMP,
318 dest, sz::byte); }
319
320 void jmp(CodeAddress dest) {
321 xedInstrRelBr(XED_ICLASS_JMP, dest, sz::dword);
322 }
323
324 void call(CodeAddress dest) {
325 xedInstrRelBr(XED_ICLASS_CALL_NEAR, dest, sz::dword);
326 }
327
328 void jcc(ConditionCode cond, CodeAddress dest) {
329 xedInstrRelBr(ccToXedJump(cond), dest, sz::dword);
330 }
331
332 void jcc8(ConditionCode cond, CodeAddress dest) {
333 xedInstrRelBr(ccToXedJump(cond), dest, sz::byte);
334 }
335
336 using X64AssemblerBase::call;
337 using X64AssemblerBase::jmp;
338 using X64AssemblerBase::jmp8;
339 using X64AssemblerBase::jcc;
340 using X64AssemblerBase::jcc8;
341
342 void setcc(int cc, Reg8 byteReg) {
343 xedInstrR(ccToXedSetCC(cc), byteReg);
344 }
345
346 void psllq(Immed i, RegXMM r) { xedInstrIR(XED_ICLASS_PSLLQ, i, r,
347 sz::byte); }
348 void psrlq(Immed i, RegXMM r) { xedInstrIR(XED_ICLASS_PSRLQ, i, r,
349 sz::byte); }
350
351 void movq_rx(Reg64 rsrc, RegXMM rdest) {
352 xedInstrRR(XED_ICLASS_MOVQ, rsrc, rdest);
353 }
354 void movq_xr(RegXMM rsrc, Reg64 rdest) {
355 xedInstrRR(XED_ICLASS_MOVQ, rsrc, rdest);
356 }
357
358 void addsd(RegXMM src, RegXMM srcdest) {
359 xedInstrRR(XED_ICLASS_ADDSD, srcdest, src);
360 }
361 void mulsd(RegXMM src, RegXMM srcdest) {
362 xedInstrRR(XED_ICLASS_MULSD, srcdest, src);
363 }
364 void subsd(RegXMM src, RegXMM srcdest) {
365 xedInstrRR(XED_ICLASS_SUBSD, srcdest, src);
366 }
367 void pxor(RegXMM src, RegXMM srcdest) {
368 xedInstrRR(XED_ICLASS_PXOR, srcdest, src);
369 }
370 void cvtsi2sd(Reg64 src, RegXMM dest) {
371 xedInstrRR(XED_ICLASS_CVTSI2SD, src, dest);
372 }
373 void cvtsi2sd(MemoryRef m, RegXMM dest) {
374 xedInstrMR(XED_ICLASS_CVTSI2SD, m, dest);
375 }
376 void ucomisd(RegXMM l, RegXMM r) {
377 xedInstrRR(XED_ICLASS_UCOMISD, l, r);
378 }
379 void sqrtsd(RegXMM src, RegXMM dest) {
380 xedInstrRR(XED_ICLASS_SQRTSD, dest, src);
381 }
382
383 void divsd(RegXMM src, RegXMM srcdest) {
384 xedInstrRR(XED_ICLASS_DIVSD, srcdest, src);
385 }
386 void cvttsd2siq(RegXMM src, Reg64 dest) {
387 xedInstrRR(XED_ICLASS_CVTTSD2SI, src, dest);
388 }
389
390 private:
391 // XED conditional jump conversion functions
392 #define CC_TO_XED_ARRAY(xed_instr) { \
393 XED_ICLASS_##xed_instr##O, /*CC_O */ \
394 XED_ICLASS_##xed_instr##NO, /*CC_NO */ \
395 XED_ICLASS_##xed_instr##B, /*CC_B, CC_NAE */ \
396 XED_ICLASS_##xed_instr##NB, /*CC_AE, CC_NB, CC_NC */ \
397 XED_ICLASS_##xed_instr##Z, /*CC_E, CC_Z */ \
398 XED_ICLASS_##xed_instr##NZ, /*CC_NE, CC_NZ */ \
399 XED_ICLASS_##xed_instr##BE, /*CC_BE, CC_NA */ \
400 XED_ICLASS_##xed_instr##NBE, /*CC_A, CC_NBE */ \
401 XED_ICLASS_##xed_instr##S, /*CC_S */ \
402 XED_ICLASS_##xed_instr##NS, /*CC_NS */ \
403 XED_ICLASS_##xed_instr##P, /*CC_P */ \
404 XED_ICLASS_##xed_instr##NP, /*CC_NP */ \
405 XED_ICLASS_##xed_instr##L, /*CC_L, CC_NGE */ \
406 XED_ICLASS_##xed_instr##NL, /*CC_GE, CC_NL */ \
407 XED_ICLASS_##xed_instr##LE, /*CC_LE, CC_NG */ \
408 XED_ICLASS_##xed_instr##NLE /*CC_G, CC_NLE */ \
409 }
410
411 ALWAYS_INLINE
412 xed_iclass_enum_t ccToXedJump(ConditionCode c) {
413 assertx(c != CC_None);
414 static const xed_iclass_enum_t jumps[] = CC_TO_XED_ARRAY(J);
415 return jumps[(int)c];
416 }
417
418 ALWAYS_INLINE
419 xed_iclass_enum_t ccToXedSetCC(int c) {
420 assertx(c != -1);
421 static const xed_iclass_enum_t setccs[] = CC_TO_XED_ARRAY(SET);
422 return setccs[c];
423 }
424
425 ALWAYS_INLINE
426 xed_iclass_enum_t ccToXedCMov(ConditionCode c) {
427 assertx(c != CC_None);
428 static const xed_iclass_enum_t cmovs[] = CC_TO_XED_ARRAY(CMOV);
429 return cmovs[(int)c];
430 }
431
432 // XED emit functions
433
434 template<typename F> ALWAYS_INLINE
435 uint32_t xedEmitImpl(xed_iclass_enum_t instr, CodeAddress destination,
436 F xedFunc) {
437 xed_encoder_instruction_t instruction;
438 xed_encoder_request_t request;
439 uint32_t encodedSize = 0;
440 xed_error_enum_t xedError;
441 xed_bool_t convert_ok;
442
443 xedFunc(&instruction);
444
445 xed_encoder_request_zero(&request);
446 convert_ok = xed_convert_to_encoder_request(&request, &instruction);
447 always_assert(convert_ok && "Unable to convert instruction"
448 " to encoder request");
449 xedError = xed_encode(&request, destination, XED_MAX_INSTRUCTION_BYTES,
450 &encodedSize);
451 always_assert_flog(xedError == XED_ERROR_NONE,
452 "XED: Error when encoding {}(): {}",
453 xed_iclass_enum_t2str(instr),
454 xed_error_enum_t2str(xedError));
455 return encodedSize;
456 }
457
458 ALWAYS_INLINE
459 uint32_t xedEmit(xed_iclass_enum_t instr, xed_uint_t effOperandSizeBits,
460 CodeAddress destination = nullptr) {
461 auto size = xedEmitImpl(instr, destination ? destination : dest(),
462 [&](xed_encoder_instruction_t* i) {
463 xed_inst0(i, kXedState, instr,
464 effOperandSizeBits);
465 });
466 if (!destination) codeBlock.moveFrontier(size);
467 return size;
468 }
469
470 ALWAYS_INLINE
471 uint32_t xedEmit(xed_iclass_enum_t instr,
472 const xed_encoder_operand_t& op,
473 xed_uint_t effOperandSizeBits = 0,
474 CodeAddress destination = nullptr) {
475 auto size = xedEmitImpl(instr, destination ? destination : dest(),
476 [&](xed_encoder_instruction_t* i) {
477 xed_inst1(i, kXedState, instr,
478 effOperandSizeBits, op);
479 });
480 if (!destination) codeBlock.moveFrontier(size);
481 return size;
482 }
483
484 ALWAYS_INLINE
485 uint32_t xedEmit(xed_iclass_enum_t instr,
486 const xed_encoder_operand_t& op_1,
487 const xed_encoder_operand_t& op_2,
488 xed_uint_t effOperandSizeBits = 0,
489 CodeAddress destination = nullptr) {
490 auto size = xedEmitImpl(instr, destination ? destination : dest(),
491 [&](xed_encoder_instruction_t* i) {
492 xed_inst2(i, kXedState, instr,
493 effOperandSizeBits, op_1, op_2);
494 });
495 if (!destination) codeBlock.moveFrontier(size);
496 return size;
497 }
498
499 ALWAYS_INLINE
500 uint32_t xedEmit(xed_iclass_enum_t instr,
501 const xed_encoder_operand_t& op_1,
502 const xed_encoder_operand_t& op_2,
503 const xed_encoder_operand_t& op_3,
504 xed_uint_t effOperandSizeBits = 0,
505 CodeAddress destination = nullptr) {
506 auto size = xedEmitImpl(instr, destination ? destination : dest(),
507 [&](xed_encoder_instruction_t* i) {
508 xed_inst3(i, kXedState, instr,
509 effOperandSizeBits,
510 op_1, op_2, op_3);
511 });
512 if (!destination) codeBlock.moveFrontier(size);
513 return size;
514 }
515
516 public:
517 static constexpr auto kInt3Size = sz::byte;
518 static constexpr auto kUd2Size = sz::word;
519
520 void emitInt3s(int n) {
521 if (n == 0) return;
522 static auto const instr = [&]{
523 uint8_t int3;
524 xedEmit(XED_ICLASS_INT3, sz::byte,
525 reinterpret_cast<CodeAddress>(&int3));
526 return int3;
527 }();
528 for (auto i = 0; i < n; ++i) {
529 byte(instr);
530 }
531 }
532
533 void emitNop(int n) {
534 if (n == 0) return;
535 static const xed_iclass_enum_t nops[] = {
536 XED_ICLASS_INVALID,
537 XED_ICLASS_NOP,
538 XED_ICLASS_NOP2,
539 XED_ICLASS_NOP3,
540 XED_ICLASS_NOP4,
541 XED_ICLASS_NOP5,
542 XED_ICLASS_NOP6,
543 XED_ICLASS_NOP7,
544 XED_ICLASS_NOP8,
545 XED_ICLASS_NOP9,
546 };
547 // While n >= 9, emit 9 byte NOPs
548 while (n >= 9) {
549 xedInstr(XED_ICLASS_NOP9, sz::nosize);
550 n -= 9;
551 }
552 // Emit remaining NOPs (if any)
553 if (n) {
554 xedInstr(nops[n], sz::nosize);
555 }
556 }
557
558 void pad() {
559 auto remaining = available();
560 if (remaining == 0) return;
561 static auto const instrs = [&]{
562 struct {
563 uint8_t int3;
564 uint16_t ud2;
565 } data;
566 xedEmit(XED_ICLASS_INT3, sz::nosize,
567 reinterpret_cast<CodeAddress>(&data.int3));
568 xedEmit(XED_ICLASS_UD2, sz::nosize,
569 reinterpret_cast<CodeAddress>(&data.ud2));
570 return data;
571 }();
572 while (remaining >= kUd2Size) {
573 word(instrs.ud2);
574 remaining -= kUd2Size;
575 }
576 while (remaining >= kInt3Size) {
577 byte(instrs.int3);
578 remaining -= kInt3Size;
579 }
580 }
581
582 ALWAYS_INLINE
583 XedAssembler& prefix(const MemoryRef& mr) {
584 return *this;
585 }
586
587 public:
588 /*
589 * The following functions use a naming convention for an older API
590 * to the assembler; conditional loads and moves haven't yet been
591 * ported.
592 */
593
594 // CMOVcc [rbase + off], rdest
595 inline void cload_reg64_disp_reg64(ConditionCode cc, Reg64 rbase,
596 int off, Reg64 rdest) {
597 MemoryRef m(DispReg(rbase, off));
598 xedInstrMR(ccToXedCMov(cc), m, rdest);
599 }
600 inline void cload_reg64_disp_reg32(ConditionCode cc, Reg64 rbase,
601 int off, Reg32 rdest) {
602 MemoryRef m(DispReg(rbase, off));
603 xedInstrMR(ccToXedCMov(cc), m, rdest);
604 }
605 inline void cmov_reg64_reg64(ConditionCode cc, Reg64 rsrc, Reg64 rdest) {
606 xedInstrRR(ccToXedCMov(cc), rsrc, rdest);
607 }
608
609 private:
610 /*
611 * The following section contains conversion methods that take a Reg8/32/64,
612 * RegXMM, MemoryRef, RipRelative struct and convert it to a
613 * xed_encoder_operand_t.
614 */
615
616 static constexpr int bytesToBits(int sz) {
617 return (sz << 3);
618 }
619
620 static constexpr int bitsToBytes(int bits) {
621 return (bits >> 3);
622 }
623
624 union XedImmValue {
625 int8_t b;
626 uint8_t ub;
627 int16_t w;
628 int32_t l;
629 int64_t q;
630 uint64_t uq;
631
632 template<typename immtype>
633 XedImmValue(const immtype& imm, int immSize) {
634 uq = 0;
635 switch (immSize) {
636 case sz::byte:
637 b = imm.b(); break;
638 case sz::word:
639 w = imm.w(); break;
640 case sz::dword:
641 l = imm.l(); break;
642 case sz::qword:
643 q = imm.q(); break;
644 }
645 }
646 };
647
648 xed_reg_enum_t xedFromReg(const Reg64& reg) {
649 return xed_reg_enum_t(int(reg) + XED_REG_RAX);
650 }
651
652 xed_reg_enum_t xedFromReg(const Reg32& reg) {
653 return xed_reg_enum_t(int(reg) + XED_REG_EAX);
654 }
655
656 xed_reg_enum_t xedFromReg(const Reg16& reg) {
657 return xed_reg_enum_t(int(reg) + XED_REG_AX);
658 }
659
660 xed_reg_enum_t xedFromReg(const Reg8& reg) {
661 auto regid = int(reg);
662 if ((regid & 0x80) == 0) {
663 return xed_reg_enum_t(regid + XED_REG_AL);
664 }
665 return xed_reg_enum_t((regid - 0x84) + XED_REG_AH);
666 }
667
668 xed_reg_enum_t xedFromReg(const RegXMM& reg) {
669 return xed_reg_enum_t(int(reg) + XED_REG_XMM0);
670 }
671
672 int getDisplSize(intptr_t value) {
673 if (value == 0) return sz::nosize;
674 return deltaFits(value, sz::byte) ? sz::byte : sz::dword;
675 }
676
677 xed_enc_displacement_t xedDispFromValue(intptr_t value, int size) {
678 switch (size) {
679 case sz::nosize: return {0, 0};
680 case sz::byte: return {(xed_uint64_t)safe_cast<int8_t>(value),
681 (xed_uint32_t)bytesToBits(size)};
682 default: return {(xed_uint64_t)safe_cast<int32_t>(value),
683 (xed_uint32_t)bytesToBits(size)};
684 }
685 }
686
687 xed_enc_displacement_t xedDispFromValue(intptr_t value) {
688 return xedDispFromValue(value, getDisplSize(value));
689 }
690
691 template<typename regtype>
692 xed_encoder_operand_t toXedOperand(const regtype& reg) {
693 return xed_reg(xedFromReg(reg));
694 }
695
696 xed_encoder_operand_t toXedOperand(xed_reg_enum_t reg) {
697 return xed_reg(reg);
698 }
699
700 xed_encoder_operand_t toXedOperand(const MemoryRef& m, int memSize) {
701 static const xed_reg_enum_t segmentRegs[] = {
702 XED_REG_INVALID, //Segment::DS (no segment register override)
703 XED_REG_FS, //Segment::FS
704 XED_REG_GS //Segment::GS
705 };
706 xed_reg_enum_t base = int(m.r.base) != -1 ?
707 xedFromReg(m.r.base) : XED_REG_INVALID;
708 xed_reg_enum_t index = int(m.r.index) != -1 ?
709 xedFromReg(m.r.index) : XED_REG_INVALID;
710 return xed_mem_gbisd(segmentRegs[int(m.segment)],
711 base, index, m.r.scale,
712 xedDispFromValue(m.r.disp), bytesToBits(memSize));
713 }
714
715 xed_encoder_operand_t toXedOperand(const RIPRelativeRef& r, int memSize) {
716 return xed_mem_bd(XED_REG_RIP, xedDispFromValue(r.r.disp, sz::dword),
717 bytesToBits(memSize));
718 }
719
720 xed_encoder_operand_t toXedOperand(CodeAddress address, int size) {
721 return xed_relbr(safe_cast<int32_t>((int64_t)address), bytesToBits(size));
722 }
723
724 template<typename immtype>
725 xed_encoder_operand_t toXedOperand(const immtype& immed, int immSizes) {
726 assert((immSizes != 0) &&
727 (immSizes & ~(sz::byte | sz::word | sz::dword | sz::qword)) == 0);
728 if ((immSizes & (immSizes - 1)) != 0) {
729 immSizes = reduceImmSize(immed.q(), immSizes);
730 }
731 return xed_imm0(XedImmValue(immed, immSizes).uq, bytesToBits(immSizes));
732 }
733
734 ALWAYS_INLINE
735 int reduceImmSize(int64_t value, int allowedSizes) {
736 while (allowedSizes) {
737 int crtSize = (allowedSizes & -allowedSizes);
738 if (crtSize == sz::qword || deltaFits(value, crtSize)) {
739 return crtSize;
740 }
741 allowedSizes ^= crtSize;
742 }
743 assertx(false && "Could not find an optimal size for Immed");
744 return sz::nosize;
745 }
746
747 /*
748 * Cache sizes for instruction types in a certain xedInstr context.
749 * This helps with emitting instructions where you need to know in advance
750 * the length of the instruction being emitted (such as when one of
751 * the operands is a RIPRelativeRef) by caching the size of the instruction
752 * and removing the need to call xedEmit twice each time (once to get
753 * the size, and once to actually emit the instruction).
754 */
755 typedef tbb::concurrent_unordered_map<int32_t, uint32_t> XedLenCache;
756
757 template<typename F> ALWAYS_INLINE
758 uint32_t xedCacheLen(XedLenCache* lenCache, F xedFunc, uint32_t key) {
759 auto res = lenCache->find(key);
760 if (res != lenCache->end()) {
761 return res->second;
762 }
763 auto instrLen = xedFunc();
764 lenCache->insert({key, instrLen});
765 return instrLen;
766 }
767
768 static constexpr uint32_t xedLenCacheKey(xed_iclass_enum_t instr,
769 uint32_t size) {
770 // 16 bits should fit a xed_iclass_enum_t value (there are currently ~1560
771 // distinct values).
772 return uint32_t(instr) | (size << 16);
773 }
774
775 // XEDInstr* wrappers
776
777 #define XED_WRAP_IMPL() \
778 XED_WRAP_X(64) \
779 XED_WRAP_X(32) \
780 XED_WRAP_X(16) \
781 XED_WRAP_X(8)
782
783 // instr(reg)
784
785 #define XED_INSTR_WRAPPER_IMPL(bitsize) \
786 ALWAYS_INLINE \
787 void xedInstrR(xed_iclass_enum_t instr, const Reg##bitsize& r) { \
788 xedEmit(instr, toXedOperand(r), bitsize); \
789 }
790
791 #define XED_WRAP_X XED_INSTR_WRAPPER_IMPL
792 XED_WRAP_IMPL()
793 #undef XED_WRAP_X
794
795 // instr(imm, reg)
796
797 #define XED_INSTIR_WRAPPER_IMPL(bitsize) \
798 ALWAYS_INLINE \
799 void xedInstrIR(xed_iclass_enum_t instr, const Immed& i, \
800 const Reg##bitsize& r, \
801 int immSize = bitsToBytes(bitsize)) { \
802 xedEmit(instr, toXedOperand(r), toXedOperand(i, immSize), \
803 bitsize); \
804 } \
805
806 #define XED_WRAP_X XED_INSTIR_WRAPPER_IMPL
807 XED_WRAP_IMPL()
808 #undef XED_WRAP_X
809
810 ALWAYS_INLINE
811 void xedInstrIR(xed_iclass_enum_t instr, const Immed64& i, const Reg64& r) {
812 xedEmit(instr, toXedOperand(r), toXedOperand(i, sz::qword),
813 bytesToBits(sz::qword));
814 }
815
816 ALWAYS_INLINE
817 void xedInstrIR(xed_iclass_enum_t instr, const Immed64& i, const Reg64& r,
818 int immSize) {
819 xedEmit(instr, toXedOperand(r), toXedOperand(i, immSize),
820 bytesToBits(sz::qword));
821 }
822
823 ALWAYS_INLINE
824 void xedInstrIR(xed_iclass_enum_t instr, const Immed& i,
825 const RegXMM& r, int immSize) {
826 xedEmit(instr, toXedOperand(r), toXedOperand(i, immSize));
827 }
828
829 // instr(reg, reg)
830
831 #define XED_INSTRR_WRAPPER_IMPL(bitsize) \
832 ALWAYS_INLINE \
833 void xedInstrRR(xed_iclass_enum_t instr, const Reg##bitsize& r1, \
834 const Reg##bitsize& r2) { \
835 xedEmit(instr, toXedOperand(r2), toXedOperand(r1), bitsize); \
836 }
837
838 #define XED_WRAP_X XED_INSTRR_WRAPPER_IMPL
839 XED_WRAP_IMPL()
840 #undef XED_WRAP_X
841
842 ALWAYS_INLINE
843 void xedInstrRR_CL(xed_iclass_enum_t instr, const Reg64& r) {
844 xedEmit(instr, toXedOperand(r), toXedOperand(XED_REG_CL),
845 bytesToBits(sz::qword));
846 }
847
848 ALWAYS_INLINE
849 void xedInstrRR(xed_iclass_enum_t instr, const Reg8& r1, const Reg32& r2,
850 int size = sz::dword) {
851 xedEmit(instr, toXedOperand(r2), toXedOperand(r1), bytesToBits(size));
852 }
853
854 ALWAYS_INLINE
855 void xedInstrRR(xed_iclass_enum_t instr, const Reg16& r1, const Reg32& r2,
856 int size = sz::dword) {
857 xedEmit(instr, toXedOperand(r2), toXedOperand(r1), bytesToBits(size));
858 }
859
860 ALWAYS_INLINE
861 void xedInstrRR(xed_iclass_enum_t instr, const Reg8& r1, const Reg64& r2,
862 int size = sz::qword) {
863 xedEmit(instr, toXedOperand(r2), toXedOperand(r1), bytesToBits(size));
864 }
865
866 ALWAYS_INLINE
867 void xedInstrRR(xed_iclass_enum_t instr, const Reg64& r1, const RegXMM& r2) {
868 xedEmit(instr, toXedOperand(r2), toXedOperand(r1));
869 }
870
871 ALWAYS_INLINE
872 void xedInstrRR(xed_iclass_enum_t instr, const RegXMM& r1, const Reg64& r2) {
873 xedEmit(instr, toXedOperand(r2), toXedOperand(r1));
874 }
875
876 // Most instr(xmm_1, xmm_2) instructions take operands in reverse order
877 // compared to instr(reg_1, reg_2): source and destination are swapped
878
879 ALWAYS_INLINE
880 void xedInstrRR(xed_iclass_enum_t instr, const RegXMM& r1, const RegXMM& r2) {
881 xedEmit(instr, toXedOperand(r1), toXedOperand(r2));
882 }
883
884 // instr(imm)
885
886 ALWAYS_INLINE
887 void xedInstrI(xed_iclass_enum_t instr, const Immed& i, int immSize) {
888 xed_encoder_operand_t op = toXedOperand(i, immSize);
889 xedEmit(instr, op, op.width_bits);
890 }
891
892 // instr(mem)
893
894 ALWAYS_INLINE
895 void xedInstrM(xed_iclass_enum_t instr, const MemoryRef& m,
896 int size = sz::qword) {
897 xedEmit(instr, toXedOperand(m, size), bytesToBits(size));
898 }
899
900 ALWAYS_INLINE
901 void xedInstrM(xed_iclass_enum_t instr, RIPRelativeRef m,
902 int size = sz::qword) {
903 static XedLenCache lenCache;
904 auto instrLen = xedCacheLen(
905 &lenCache,
906 [&]() {
907 return xedEmit(instr, toXedOperand(nullrip, size),
908 bytesToBits(size), dest());
909 }, xedLenCacheKey(instr, 0));
910 m.r.disp -= ((int64_t)frontier() + (int64_t)instrLen);
911 xedEmit(instr, toXedOperand(m, size), bytesToBits(size));
912 }
913
914 // instr(imm, mem)
915
916 ALWAYS_INLINE
917 void xedInstrIM(xed_iclass_enum_t instr, const Immed& i, const MemoryRef& m,
918 int size = sz::qword) {
919 assert(size && (size & (size - 1)) == 0);
920 xedEmit(instr, toXedOperand(m, size), toXedOperand(i, size),
921 bytesToBits(size));
922 }
923
924 ALWAYS_INLINE
925 void xedInstrIM(xed_iclass_enum_t instr, const Immed& i, const MemoryRef& m,
926 int immSize, int memSize) {
927 xedEmit(instr, toXedOperand(m, memSize), toXedOperand(i, immSize),
928 bytesToBits(memSize));
929 }
930
931 // instr(mem, reg)
932
933 #define XED_INSTMR_WRAPPER_IMPL(bitsize) \
934 ALWAYS_INLINE \
935 void xedInstrMR(xed_iclass_enum_t instr, const MemoryRef& m, \
936 const Reg##bitsize& r, \
937 int memSize = bitsToBytes(bitsize)) { \
938 xedEmit(instr, toXedOperand(r), toXedOperand(m, memSize), bitsize); \
939 } \
940 \
941 ALWAYS_INLINE \
942 void xedInstrMR(xed_iclass_enum_t instr, RIPRelativeRef m, \
943 const Reg##bitsize& r) { \
944 static XedLenCache lenCache; \
945 auto instrLen = xedCacheLen( \
946 &lenCache, \
947 [&] { \
948 return xedEmit( \
949 instr, toXedOperand(r), \
950 toXedOperand(nullrip, \
951 bitsToBytes(bitsize)), \
952 bitsize, dest()); \
953 }, xedLenCacheKey(instr, 0)); \
954 m.r.disp -= ((int64_t)frontier() + (int64_t)instrLen); \
955 xedEmit(instr, toXedOperand(r), \
956 toXedOperand(m, bitsToBytes(bitsize)), bitsize); \
957 }
958
959 #define XED_WRAP_X XED_INSTMR_WRAPPER_IMPL
960 XED_WRAP_IMPL()
961 #undef XED_WRAP_X
962
963 ALWAYS_INLINE
964 void xedInstrMR(xed_iclass_enum_t instr, const MemoryRef& m,
965 const RegXMM& r, int memSize = sz::qword) {
966 xedEmit(instr, toXedOperand(r), toXedOperand(m, memSize));
967 }
968
969 ALWAYS_INLINE
970 void xedInstrMR(xed_iclass_enum_t instr, RIPRelativeRef m,
971 const RegXMM& r, int memSize = sz::qword) {
972 static XedLenCache lenCache;
973 auto instrLen = xedCacheLen(
974 &lenCache,
975 [&]() {
976 return xedEmit(
977 instr, toXedOperand(r),
978 toXedOperand(nullrip, memSize),
979 0, dest());
980 }, xedLenCacheKey(instr, 0));
981 m.r.disp -= ((int64_t)frontier() + (int64_t)instrLen);
982 xedEmit(instr, toXedOperand(r), toXedOperand(m, memSize));
983 }
984
985 // instr(reg, mem)
986
987 #define XED_INSTRM_WRAPPER_IMPL(bitsize) \
988 ALWAYS_INLINE \
989 void xedInstrRM(xed_iclass_enum_t instr, const Reg##bitsize& r, \
990 const MemoryRef& m) { \
991 xedEmit(instr, toXedOperand(m, bitsToBytes(bitsize)), \
992 toXedOperand(r), bitsize); \
993 }
994
995 #define XED_WRAP_X XED_INSTRM_WRAPPER_IMPL
996 XED_WRAP_IMPL()
997 #undef XED_WRAP_X
998
999 ALWAYS_INLINE
1000 void xedInstrRM(xed_iclass_enum_t instr, const RegXMM& r,
1001 const MemoryRef& m, int memSize = sz::qword) {
1002 xedEmit(instr, toXedOperand(m, memSize), toXedOperand(r));
1003 }
1004
1005 // instr(xmm, xmm, imm)
1006
1007 ALWAYS_INLINE
1008 void xedInstrIRR(xed_iclass_enum_t instr, const RegXMM& r1, const RegXMM& r2,
1009 const Immed& i, int immSize) {
1010 xedEmit(instr, toXedOperand(r1), toXedOperand(r2),
1011 toXedOperand(i, immSize));
1012 }
1013
1014 // instr(relbr)
1015
1016 void xedInstrRelBr(xed_iclass_enum_t instr,
1017 CodeAddress destination, int size) {
1018 static XedLenCache lenCache;
1019 auto instrLen = xedCacheLen(
1020 &lenCache,
1021 [&]() {
1022 return xedEmit(instr, toXedOperand((CodeAddress)0,
1023 size), 0, dest());
1024 }, xedLenCacheKey(instr, size));
1025 auto target = destination - (frontier() + instrLen);
1026 xedEmit(instr, toXedOperand((CodeAddress)target, size));
1027 }
1028
1029 // instr()
1030
1031 ALWAYS_INLINE
1032 void xedInstr(xed_iclass_enum_t instr, int size = sz::qword) {
1033 xedEmit(instr, bytesToBits(size));
1034 }
1035 };
1036
1037 }