?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add a HHIR-level peephole optimization to reorder CheckTypes
[hiphop-php.git] / hphp / util / asm-x64.h
blob2d4b0bf9711911213a4121155e2300af0dd698a5
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 */
16 #pragma once
17
18 #include <type_traits>
19 #include <memory>
20
21 #include "hphp/util/atomic.h"
22 #include "hphp/util/data-block.h"
23 #include "hphp/util/immed.h"
24 #include "hphp/util/safe-cast.h"
25 #include "hphp/util/trace.h"
26
27 /*
28 * An experimental macro assembler for x64, that strives for low coupling to
29 * the runtime environment.
30 *
31 * There are more complete assemblers out there; if you use this one
32 * yourself, expect not to find all the instructions you wanted to use. You
33 * may have to go spelunking in the Intel manuals:
34 *
35 * http://www.intel.com/products/processor/manuals/
36 *
37 * If you're looking for something more fully baked, here are some options
38 * to consider:
39 *
40 * 1. Nanojit or llvm, both of which translate abstract virtual machine
41 * instructions to the native target architecture, or
42 * 2. The embedded assemblers from v8, the Sun JVM, etc.
43 */
44
45 /*
46 * Some members cannot be const because their values aren't known in
47 * an initialization list. Like the opposite of the "mutable" keyword.
48 * This declares this property to readers.
49 */
50 #define logical_const /* nothing */
51
52 namespace HPHP { namespace jit {
53
54 #define TRACEMOD ::HPHP::Trace::asmx64
55
56 //////////////////////////////////////////////////////////////////////
57
58 struct MemoryRef;
59 struct RIPRelativeRef;
60 struct ScaledIndex;
61 struct ScaledIndexDisp;
62 struct DispReg;
63
64 const uint8_t kOpsizePrefix = 0x66;
65
66 enum Segment : uint8_t { DS = 0, FS, GS };
67
68 struct Reg64 {
69 explicit constexpr Reg64(int rn) : rn(rn) {}
70
71 // Integer conversion is allowed but only explicitly. (It's not
72 // unusual to want to printf registers, etc. Just cast it first.)
73 explicit constexpr operator int() const { return rn; }
74
75 MemoryRef operator[](intptr_t disp) const;
76 MemoryRef operator[](Reg64) const;
77 MemoryRef operator[](ScaledIndex) const;
78 MemoryRef operator[](ScaledIndexDisp) const;
79 MemoryRef operator[](DispReg) const;
80
81 constexpr bool operator==(Reg64 o) const { return rn == o.rn; }
82 constexpr bool operator!=(Reg64 o) const { return rn != o.rn; }
83
84 private:
85 int rn;
86 };
87
88 #define SIMPLE_REGTYPE(What) \
89 struct What { \
90 explicit constexpr What(int rn) : rn(rn) {} \
91 explicit constexpr operator int() const { return rn; } \
92 constexpr bool operator==(What o) const { return rn == o.rn; } \
93 constexpr bool operator!=(What o) const { return rn != o.rn; } \
94 private: \
95 int rn; \
96 }
97
98 SIMPLE_REGTYPE(Reg32);
99 SIMPLE_REGTYPE(Reg16);
100 SIMPLE_REGTYPE(Reg8);
101 SIMPLE_REGTYPE(RegXMM);
102 SIMPLE_REGTYPE(RegSF);
103
104 #undef SIMPLE_REGTYPE
105
106 struct RegRIP {
107 RIPRelativeRef operator[](intptr_t disp) const;
108 };
109
110 // Convert between physical registers of different sizes
111 inline Reg8 rbyte(Reg32 r) { return Reg8(int(r)); }
112 inline Reg8 rbyte(Reg64 r) { return Reg8(int(r)); }
113 inline Reg16 r16(Reg8 r) { return Reg16(int(r)); }
114 inline Reg16 r16(Reg32 r) { return Reg16(int(r)); }
115 inline Reg32 r32(Reg8 r) { return Reg32(int(r)); }
116 inline Reg32 r32(Reg16 r) { return Reg32(int(r)); }
117 inline Reg32 r32(Reg32 r) { return r; }
118 inline Reg32 r32(Reg64 r) { return Reg32(int(r)); }
119 inline Reg64 r64(Reg8 r) { return Reg64(int(r)); }
120 inline Reg64 r64(Reg16 r) { return Reg64(int(r)); }
121 inline Reg64 r64(Reg32 r) { return Reg64(int(r)); }
122 inline Reg64 r64(Reg64 r) { return r; }
123
124 //////////////////////////////////////////////////////////////////////
125
126 /*
127 * The following structures define intermediate types for various
128 * addressing modes. They overload some operators to allow using
129 * registers to look somewhat like pointers.
130 *
131 * E.g. rax[rbx*2 + 3] or *(rax + rbx*2 + 3).
132 *
133 * These operators are not defined commutatively; the thought is it
134 * mandates the order you normally write them in a .S, but it could be
135 * changed if this proves undesirable.
136 */
137
138 // reg*x
139 struct ScaledIndex {
140 explicit ScaledIndex(Reg64 index, intptr_t scale)
141 : index(index)
142 , scale(scale)
143 {
144 assert((scale == 0x1 || scale == 0x2 || scale == 0x4 || scale == 0x8) &&
145 "Invalid index register scaling (must be 1,2,4 or 8).");
146 assert(int(index) != -1 && "invalid register");
147 }
148
149 Reg64 index;
150 intptr_t scale;
151 };
152
153 // reg*x + disp
154 struct ScaledIndexDisp {
155 explicit ScaledIndexDisp(ScaledIndex si, intptr_t disp)
156 : si(si)
157 , disp(disp)
158 {}
159
160 ScaledIndexDisp operator+(intptr_t x) const {
161 return ScaledIndexDisp(si, disp + x);
162 }
163
164 ScaledIndexDisp operator-(intptr_t x) const {
165 return ScaledIndexDisp(si, disp - x);
166 }
167
168 ScaledIndex si;
169 intptr_t disp;
170 };
171
172 // reg+x
173 struct DispReg {
174 explicit DispReg(Reg64 base, intptr_t disp = 0)
175 : base(base)
176 , disp(disp)
177 {
178 assert(int(base) != -1 && "invalid register");
179 }
180
181 // Constructor for baseless().
182 explicit DispReg(intptr_t disp)
183 : base(-1)
184 , disp(disp)
185 {}
186
187 MemoryRef operator*() const;
188 MemoryRef operator[](intptr_t) const;
189
190 DispReg operator+(intptr_t x) const {
191 return DispReg(base, disp + x);
192 }
193
194 DispReg operator-(intptr_t x) const {
195 return DispReg(base, disp - x);
196 }
197
198 Reg64 base;
199 intptr_t disp;
200 };
201
202 // reg + reg*x + y
203 struct IndexedDispReg {
204 explicit IndexedDispReg(Reg64 base, ScaledIndex sr)
205 : base(base)
206 , index(sr.index)
207 , scale(sr.scale)
208 , disp(0)
209 {}
210
211 explicit IndexedDispReg(DispReg r)
212 : base(r.base)
213 , index(-1)
214 , scale(1)
215 , disp(r.disp)
216 {}
217
218 // Constructor for baseless()
219 explicit IndexedDispReg(ScaledIndexDisp sid)
220 : base(-1)
221 , index(sid.si.index)
222 , scale(sid.si.scale)
223 , disp(sid.disp)
224 {}
225
226 MemoryRef operator*() const;
227 MemoryRef operator[](intptr_t disp) const;
228
229 IndexedDispReg operator+(intptr_t disp) const {
230 auto ret = *this;
231 ret.disp += disp;
232 return ret;
233 }
234
235 IndexedDispReg operator-(intptr_t disp) const {
236 auto ret = *this;
237 ret.disp -= disp;
238 return ret;
239 }
240
241 Reg64 base;
242 Reg64 index;
243 int scale;
244 intptr_t disp; // TODO #4613274: should be int32_t
245 };
246
247 // rip+x
248 struct DispRIP {
249 explicit constexpr DispRIP(intptr_t disp) : disp(disp) {}
250
251 RIPRelativeRef operator*() const;
252 RIPRelativeRef operator[](intptr_t x) const;
253
254 DispRIP operator+(intptr_t x) const {
255 return DispRIP(disp + x);
256 }
257
258 DispRIP operator-(intptr_t x) const {
259 return DispRIP(disp - x);
260 }
261
262 bool operator==(DispRIP o) const { return disp == o.disp; }
263 bool operator!=(DispRIP o) const { return disp != o.disp; }
264
265 intptr_t disp;
266 };
267
268 // *(rip + x)
269 struct RIPRelativeRef {
270 explicit constexpr RIPRelativeRef(DispRIP r) : r(r) {}
271 DispRIP r;
272
273 bool operator==(const RIPRelativeRef& o) const { return r == o.r; }
274 bool operator!=(const RIPRelativeRef& o) const { return r != o.r; }
275 };
276
277 //////////////////////////////////////////////////////////////////////
278
279 namespace reg {
280 constexpr Reg64 rax(0);
281 constexpr Reg64 rcx(1);
282 constexpr Reg64 rdx(2);
283 constexpr Reg64 rbx(3);
284 constexpr Reg64 rsp(4);
285 constexpr Reg64 rbp(5);
286 constexpr Reg64 rsi(6);
287 constexpr Reg64 rdi(7);
288
289 constexpr Reg64 r8 (8);
290 constexpr Reg64 r9 (9);
291 constexpr Reg64 r10(10);
292 constexpr Reg64 r11(11);
293 constexpr Reg64 r12(12);
294 constexpr Reg64 r13(13);
295 constexpr Reg64 r14(14);
296 constexpr Reg64 r15(15);
297
298 constexpr RegRIP rip = RegRIP();
299
300 constexpr Reg32 eax (0);
301 constexpr Reg32 ecx (1);
302 constexpr Reg32 edx (2);
303 constexpr Reg32 ebx (3);
304 constexpr Reg32 esp (4);
305 constexpr Reg32 ebp (5);
306 constexpr Reg32 esi (6);
307 constexpr Reg32 edi (7);
308 constexpr Reg32 r8d (8);
309 constexpr Reg32 r9d (9);
310 constexpr Reg32 r10d(10);
311 constexpr Reg32 r11d(11);
312 constexpr Reg32 r12d(12);
313 constexpr Reg32 r13d(13);
314 constexpr Reg32 r14d(14);
315 constexpr Reg32 r15d(15);
316
317 constexpr Reg16 ax (0);
318 constexpr Reg16 cx (1);
319 constexpr Reg16 dx (2);
320 constexpr Reg16 bx (3);
321 constexpr Reg16 sp (4);
322 constexpr Reg16 bp (5);
323 constexpr Reg16 si (6);
324 constexpr Reg16 di (7);
325 constexpr Reg16 r8w (8);
326 constexpr Reg16 r9w (9);
327 constexpr Reg16 r10w(10);
328 constexpr Reg16 r11w(11);
329 constexpr Reg16 r12w(12);
330 constexpr Reg16 r13w(13);
331 constexpr Reg16 r14w(14);
332 constexpr Reg16 r15w(15);
333
334 constexpr Reg8 al (0);
335 constexpr Reg8 cl (1);
336 constexpr Reg8 dl (2);
337 constexpr Reg8 bl (3);
338 constexpr Reg8 spl (4);
339 constexpr Reg8 bpl (5);
340 constexpr Reg8 sil (6);
341 constexpr Reg8 dil (7);
342 constexpr Reg8 r8b (8);
343 constexpr Reg8 r9b (9);
344 constexpr Reg8 r10b(10);
345 constexpr Reg8 r11b(11);
346 constexpr Reg8 r12b(12);
347 constexpr Reg8 r13b(13);
348 constexpr Reg8 r14b(14);
349 constexpr Reg8 r15b(15);
350
351 // Reminder: these registers may not be mixed in any instruction
352 // using a REX prefix (i.e. anything using r8-r15, spl, bpl, sil,
353 // dil, etc).
354 constexpr Reg8 ah(0x80 | 4);
355 constexpr Reg8 ch(0x80 | 5);
356 constexpr Reg8 dh(0x80 | 6);
357 constexpr Reg8 bh(0x80 | 7);
358
359 constexpr RegXMM xmm0(0);
360 constexpr RegXMM xmm1(1);
361 constexpr RegXMM xmm2(2);
362 constexpr RegXMM xmm3(3);
363 constexpr RegXMM xmm4(4);
364 constexpr RegXMM xmm5(5);
365 constexpr RegXMM xmm6(6);
366 constexpr RegXMM xmm7(7);
367 constexpr RegXMM xmm8(8);
368 constexpr RegXMM xmm9(9);
369 constexpr RegXMM xmm10(10);
370 constexpr RegXMM xmm11(11);
371 constexpr RegXMM xmm12(12);
372 constexpr RegXMM xmm13(13);
373 constexpr RegXMM xmm14(14);
374 constexpr RegXMM xmm15(15);
375
376 #define X(x) if (r == x) return "%"#x
377 inline const char* regname(Reg64 r) {
378 X(rax); X(rbx); X(rcx); X(rdx); X(rsp); X(rbp); X(rsi); X(rdi);
379 X(r8); X(r9); X(r10); X(r11); X(r12); X(r13); X(r14); X(r15);
380 return nullptr;
381 }
382 inline const char* regname(Reg32 r) {
383 X(eax); X(ecx); X(edx); X(ebx); X(esp); X(ebp); X(esi); X(edi);
384 X(r8d); X(r9d); X(r10d); X(r11d); X(r12d); X(r13d); X(r14d); X(r15d);
385 return nullptr;
386 }
387 inline const char* regname(Reg16 r) {
388 X(ax); X(cx); X(dx); X(bx); X(sp); X(bp); X(si); X(di);
389 X(r8w); X(r9w); X(r10w); X(r11w); X(r12w); X(r13w); X(r14w); X(r15w);
390 return nullptr;
391 }
392 inline const char* regname(Reg8 r) {
393 X(al); X(cl); X(dl); X(bl); X(spl); X(bpl); X(sil); X(dil);
394 X(r8b); X(r9b); X(r10b); X(r11b); X(r12b); X(r13b); X(r14b); X(r15b);
395 X(ah); X(ch); X(dh); X(bh);
396 return nullptr;
397 }
398 inline const char* regname(RegXMM r) {
399 X(xmm0); X(xmm1); X(xmm2); X(xmm3); X(xmm4); X(xmm5); X(xmm6);
400 X(xmm7); X(xmm8); X(xmm9); X(xmm10); X(xmm11); X(xmm12); X(xmm13);
401 X(xmm14); X(xmm15);
402 return nullptr;
403 }
404 inline const char* regname(RegSF /*r*/) {
405 return "%flags";
406 }
407 #undef X
408
409 }
410
411 enum class RoundDirection : ssize_t {
412 nearest = 0,
413 floor = 1,
414 ceil = 2,
415 truncate = 3,
416 };
417
418 const char* show(RoundDirection);
419
420 enum class ComparisonPred : uint8_t {
421 // True if...
422 eq_ord = 0, // ...operands are ordered AND equal
423 ne_unord = 4, // ...operands are unordered OR unequal
424 };
425
426 enum ConditionCode {
427 CC_None = -1,
428 CC_O = 0x00,
429 CC_NO = 0x01,
430
431 CC_B = 0x02,
432 CC_NAE = 0x02,
433 CC_AE = 0x03,
434 CC_NB = 0x03,
435 CC_NC = 0x03,
436
437 CC_E = 0x04,
438 CC_Z = 0x04,
439 CC_NE = 0x05,
440 CC_NZ = 0x05,
441
442 CC_BE = 0x06,
443 CC_NA = 0x06,
444 CC_A = 0x07,
445 CC_NBE = 0x07,
446
447 CC_S = 0x08,
448 CC_NS = 0x09,
449
450 CC_P = 0x0A,
451 CC_NP = 0x0B,
452
453 CC_L = 0x0C,
454 CC_NGE = 0x0C,
455 CC_GE = 0x0D,
456 CC_NL = 0x0D,
457
458 CC_LE = 0x0E,
459 CC_NG = 0x0E,
460 CC_G = 0x0F,
461 CC_NLE = 0x0F,
462 };
463
464 // names of condition codes, indexable by the ConditionCode enum value.
465 extern const char* cc_names[];
466
467 inline ConditionCode ccNegate(ConditionCode c) {
468 return ConditionCode(int(c) ^ 1); // And you thought x86 was irregular!
469 }
470
471 // *(reg + x)
472 struct MemoryRef {
473 explicit MemoryRef(DispReg dr, Segment s = DS) : r(dr), segment(s) {}
474 explicit MemoryRef(IndexedDispReg idr, Segment s = DS) : r(idr), segment(s) {}
475 IndexedDispReg r;
476 Segment segment;
477 };
478
479 /*
480 * Simple wrapper over a Segment value used to obtain MemoryRefs that have
481 * MemoryRef::segment set to something different than the default (DS) value.
482 */
483 struct SegReg {
484 explicit constexpr SegReg(Segment seg) : seg(seg) {};
485 MemoryRef operator[](const IndexedDispReg& idr) {
486 return MemoryRef(idr, seg);
487 }
488 MemoryRef operator[](const ScaledIndexDisp& sid) {
489 return MemoryRef(IndexedDispReg(sid), seg);
490 }
491 MemoryRef operator[](const DispReg& dr) {
492 return MemoryRef(dr, seg);
493 }
494 MemoryRef operator[](const intptr_t disp) {
495 return MemoryRef(DispReg(disp), seg);
496 }
497 Segment seg;
498 };
499
500 #ifdef HAVE_LIBXED
501 #define NEW_X64_ASM(var, cb) \
502 std::unique_ptr<X64AssemblerBase> _assembler( \
503 RuntimeOption::EvalUseXedAssembler ? \
504 (X64AssemblerBase*)new XedAssembler(cb) : \
505 (X64AssemblerBase*)new X64Assembler(cb)); \
506 X64AssemblerBase& var = *_assembler
507 #else
508 #define NEW_X64_ASM(var, cb) X64Assembler var(cb)
509 #endif
510
511 struct Label;
512
513 struct X64AssemblerBase {
514 protected:
515 friend struct Label;
516
517 /*
518 * Type for register numbers, independent of the size we're going to
519 * be using it as. Also, the same register number may mean different
520 * physical registers for different instructions (e.g. xmm0 and rax
521 * are both 0). Only for internal use in X64Assembler.
522 */
523 enum class RegNumber : int {};
524 static const RegNumber noreg = RegNumber(-1);
525
526 public:
527 explicit X64AssemblerBase(CodeBlock& cb) : codeBlock(cb) {}
528 virtual ~X64AssemblerBase() {}
529
530 CodeBlock& code() const { return codeBlock; }
531
532 CodeAddress base() const {
533 return codeBlock.base();
534 }
535
536 CodeAddress frontier() const {
537 return codeBlock.frontier();
538 }
539
540 CodeAddress toDestAddress(CodeAddress addr) const {
541 return codeBlock.toDestAddress(addr);
542 }
543
544 void setFrontier(CodeAddress newFrontier) {
545 codeBlock.setFrontier(newFrontier);
546 }
547
548 size_t capacity() const {
549 return codeBlock.capacity();
550 }
551
552 size_t used() const {
553 return codeBlock.used();
554 }
555
556 size_t available() const {
557 return codeBlock.available();
558 }
559
560 bool contains(CodeAddress addr) const {
561 return codeBlock.contains(addr);
562 }
563
564 bool empty() const {
565 return codeBlock.empty();
566 }
567
568 void clear() {
569 codeBlock.clear();
570 }
571
572 bool canEmit(size_t nBytes) const {
573 assert(capacity() >= used());
574 return nBytes < (capacity() - used());
575 }
576
577 #define BYTE_LOAD_OP(name) \
578 virtual void name##b(MemoryRef m, Reg8 r) = 0;
579
580 #define LOAD_OP(name) \
581 virtual void name##q(MemoryRef m, Reg64 r) = 0; \
582 virtual void name##l(MemoryRef m, Reg32 r) = 0; \
583 virtual void name##w(MemoryRef m, Reg16 r) = 0; \
584 virtual void name##q(RIPRelativeRef m, Reg64 r) = 0; \
585 BYTE_LOAD_OP(name)
586
587 #define BYTE_STORE_OP(name) \
588 virtual void name##b(Reg8 r, MemoryRef m) = 0; \
589 virtual void name##b(Immed i, MemoryRef m) = 0;
590
591 #define STORE_OP(name) \
592 virtual void name##w(Immed i, MemoryRef m) = 0; \
593 virtual void name##l(Immed i, MemoryRef m) = 0; \
594 virtual void name##w(Reg16 r, MemoryRef m) = 0; \
595 virtual void name##l(Reg32 r, MemoryRef m) = 0; \
596 virtual void name##q(Reg64 r, MemoryRef m) = 0; \
597 BYTE_STORE_OP(name)
598
599 #define BYTE_REG_OP(name) \
600 virtual void name##b(Reg8 r1, Reg8 r2) = 0; \
601 virtual void name##b(Immed i, Reg8 r) = 0;
602
603 #define REG_OP(name) \
604 virtual void name##q(Reg64 r1, Reg64 r2) = 0; \
605 virtual void name##l(Reg32 r1, Reg32 r2) = 0; \
606 virtual void name##w(Reg16 r1, Reg16 r2) = 0; \
607 virtual void name##l(Immed i, Reg32 r) = 0; \
608 virtual void name##w(Immed i, Reg16 r) = 0; \
609 BYTE_REG_OP(name)
610
611 #define IMM64_STORE_OP(name) \
612 virtual void name##q(Immed i, MemoryRef m) = 0;
613
614 #define IMM64R_OP(name) \
615 virtual void name##q(Immed imm, Reg64 r) = 0;
616
617 #define FULL_OP(name) \
618 LOAD_OP(name) \
619 STORE_OP(name) \
620 REG_OP(name) \
621 IMM64_STORE_OP(name) \
622 IMM64R_OP(name)
623
624 LOAD_OP (load)
625 STORE_OP (store)
626 IMM64_STORE_OP (store)
627 REG_OP (mov)
628
629 FULL_OP(add)
630 FULL_OP(xor)
631 FULL_OP(sub)
632 FULL_OP(and)
633 FULL_OP(or)
634 FULL_OP(test)
635 FULL_OP(cmp)
636 FULL_OP(sbb)
637
638 #undef IMM64R_OP
639 #undef FULL_OP
640 #undef REG_OP
641 #undef STORE_OP
642 #undef LOAD_OP
643 #undef BYTE_LOAD_OP
644 #undef BYTE_STORE_OP
645 #undef BYTE_REG_OP
646 #undef IMM64_STORE_OP
647
648 virtual void movq(Immed64 imm, Reg64 r) = 0;
649 virtual void loadzbl(MemoryRef m, Reg32 r) = 0;
650 virtual void loadzwl(MemoryRef m, Reg32 r) = 0;
651 virtual void movzbl(Reg8 src, Reg32 dest) = 0;
652 virtual void movsbl(Reg8 src, Reg32 dest) = 0;
653 virtual void movzwl(Reg16 src, Reg32 dest) = 0;
654 virtual void loadsbq(MemoryRef m, Reg64 r) = 0;
655 virtual void movsbq(Reg8 src, Reg64 dest) = 0;
656 virtual void crc32q(Reg64 src, Reg64 dest) = 0;
657 virtual void lea(MemoryRef p, Reg64 reg) = 0;
658 virtual void lea(RIPRelativeRef p, Reg64 reg) = 0;
659
660 virtual void xchgq(Reg64 r1, Reg64 r2) = 0;
661 virtual void xchgl(Reg32 r1, Reg32 r2) = 0;
662 virtual void xchgb(Reg8 r1, Reg8 r2) = 0;
663
664 virtual void imul(Reg64 r1, Reg64 r2) = 0;
665
666 virtual void push(Reg64 r) = 0;
667 virtual void pushl(Reg32 r) = 0;
668 virtual void pop (Reg64 r) = 0;
669 virtual void idiv(Reg64 r) = 0;
670 virtual void incq(Reg64 r) = 0;
671 virtual void incl(Reg32 r) = 0;
672 virtual void incw(Reg16 r) = 0;
673 virtual void decq(Reg64 r) = 0;
674 virtual void decl(Reg32 r) = 0;
675 virtual void decw(Reg16 r) = 0;
676 virtual void notb(Reg8 r) = 0;
677 virtual void not(Reg64 r) = 0;
678 virtual void neg(Reg64 r) = 0;
679 virtual void negb(Reg8 r) = 0;
680 virtual void ret() = 0;
681 virtual void ret(Immed i) = 0;
682 virtual void cqo() = 0;
683 virtual void nop() = 0;
684 virtual void int3() = 0;
685 virtual void ud2() = 0;
686 virtual void pushf() = 0;
687 virtual void popf() = 0;
688 virtual void lock() = 0;
689
690 virtual void push(MemoryRef m) = 0;
691 virtual void pop (MemoryRef m) = 0;
692 virtual void prefetch(MemoryRef m) = 0;
693 virtual void incq(MemoryRef m) = 0;
694 virtual void incl(MemoryRef m) = 0;
695 virtual void incw(MemoryRef m) = 0;
696 virtual void decqlock(MemoryRef m) = 0;
697 virtual void decq(MemoryRef m) = 0;
698 virtual void decl(MemoryRef m) = 0;
699 virtual void decw(MemoryRef m) = 0;
700
701 virtual void push(Immed64 i) = 0;
702
703 virtual void movups(RegXMM x, MemoryRef m) = 0;
704 virtual void movups(MemoryRef m, RegXMM x) = 0;
705 virtual void movdqu(RegXMM x, MemoryRef m) = 0;
706 virtual void movdqu(MemoryRef m, RegXMM x) = 0;
707 virtual void movdqa(RegXMM x, RegXMM y) = 0;
708 virtual void movdqa(RegXMM x, MemoryRef m) = 0;
709 virtual void movdqa(MemoryRef m, RegXMM x) = 0;
710 virtual void movsd (RegXMM x, RegXMM y) = 0;
711 virtual void movsd (RegXMM x, MemoryRef m) = 0;
712 virtual void movsd (MemoryRef m, RegXMM x) = 0;
713 virtual void movsd (RIPRelativeRef m, RegXMM x) = 0;
714 virtual void lddqu (MemoryRef m, RegXMM x) = 0;
715 virtual void unpcklpd(RegXMM s, RegXMM d) = 0;
716
717 virtual void rorq (Immed i, Reg64 r) = 0;
718 virtual void shlq (Immed i, Reg64 r) = 0;
719 virtual void shrq (Immed i, Reg64 r) = 0;
720 virtual void sarq (Immed i, Reg64 r) = 0;
721 virtual void shll (Immed i, Reg32 r) = 0;
722 virtual void shrl (Immed i, Reg32 r) = 0;
723 virtual void shlw (Immed i, Reg16 r) = 0;
724 virtual void shrw (Immed i, Reg16 r) = 0;
725
726 virtual void shlq (Reg64 r) = 0;
727 virtual void shrq (Reg64 r) = 0;
728 virtual void sarq (Reg64 r) = 0;
729
730 virtual void roundsd (RoundDirection d, RegXMM src, RegXMM dst) = 0;
731 virtual void cmpsd(RegXMM src, RegXMM dst, ComparisonPred pred) = 0;
732
733 /*
734 * The following utility functions do more than emit specific code.
735 * (E.g. combine common idioms or patterns, smash code, etc.)
736 */
737
738 void emitImmReg(Immed64 imm, Reg64 dest) {
739 if (imm.q() == 0) {
740 // Zeros the top bits also.
741 xorl (r32(dest), r32(dest));
742 return;
743 }
744 if (LIKELY(imm.q() > 0 && imm.fits(sz::dword))) {
745 // This will zero out the high-order bits.
746 movl (imm.l(), r32(dest));
747 return;
748 }
749 movq (imm.q(), dest);
750 }
751
752 bool jmpDeltaFits(CodeAddress dest) {
753 int64_t delta = dest - (codeBlock.frontier() + 5);
754 return deltaFits(delta, sz::dword);
755 }
756
757 virtual void jmp(Reg64 r) = 0;
758 virtual void jmp(MemoryRef m) = 0;
759 virtual void jmp(RIPRelativeRef m) = 0;
760 virtual void call(Reg64 r) = 0;
761 virtual void call(MemoryRef m) = 0;
762 virtual void call(RIPRelativeRef m) = 0;
763 virtual void jmp8(CodeAddress dest) = 0;
764 virtual void jmp(CodeAddress dest) = 0;
765 virtual void call(CodeAddress dest) = 0;
766 virtual void jcc(ConditionCode cond, CodeAddress dest) = 0;
767 virtual void jcc8(ConditionCode cond, CodeAddress dest)= 0;
768
769 void jmpAuto(CodeAddress dest) {
770 auto delta = dest - (codeBlock.frontier() + 2);
771 if (deltaFits(delta, sz::byte)) {
772 jmp8(dest);
773 } else {
774 jmp(dest);
775 }
776 }
777
778 void jccAuto(ConditionCode cc, CodeAddress dest) {
779 auto delta = dest - (codeBlock.frontier() + 2);
780 if (deltaFits(delta, sz::byte)) {
781 jcc8(cc, dest);
782 } else {
783 jcc(cc, dest);
784 }
785 }
786
787 void call(Label&);
788 void jmp(Label&);
789 void jmp8(Label&);
790 void jcc(ConditionCode, Label&);
791 void jcc8(ConditionCode, Label&);
792
793 #define CCS \
794 CC(o, CC_O) \
795 CC(no, CC_NO) \
796 CC(nae, CC_NAE) \
797 CC(ae, CC_AE) \
798 CC(nb, CC_NB) \
799 CC(e, CC_E) \
800 CC(z, CC_Z) \
801 CC(ne, CC_NE) \
802 CC(nz, CC_NZ) \
803 CC(b, CC_B) \
804 CC(be, CC_BE) \
805 CC(nbe, CC_NBE) \
806 CC(s, CC_S) \
807 CC(ns, CC_NS) \
808 CC(p, CC_P) \
809 CC(np, CC_NP) \
810 CC(nge, CC_NGE) \
811 CC(g, CC_G) \
812 CC(l, CC_L) \
813 CC(ge, CC_GE) \
814 CC(nl, CC_NL) \
815 CC(ng, CC_NG) \
816 CC(le, CC_LE) \
817 CC(nle, CC_NLE)
818
819 #define CC(_nm, _code) \
820 void j ## _nm(CodeAddress dest) { jcc(_code, dest); } \
821 void j ## _nm ## 8(CodeAddress dest) { jcc8(_code, dest); } \
822 void j ## _nm(Label&); \
823 void j ## _nm ## 8(Label&);
824 CCS
825 #undef CC
826
827 #define CC(_nm, _cond) \
828 void set ## _nm(Reg8 byteReg) { \
829 setcc(_cond, byteReg); \
830 }
831 CCS
832 #undef CC
833
834 virtual void setcc(int cc, Reg8 byteReg) = 0;
835 virtual void psllq(Immed i, RegXMM r) = 0;
836 virtual void psrlq(Immed i, RegXMM r) = 0;
837 virtual void movq_rx(Reg64 rSrc, RegXMM rdest) = 0;
838 virtual void movq_xr(RegXMM rSrc, Reg64 rdest) = 0;
839 virtual void addsd(RegXMM src, RegXMM srcdest) = 0;
840 virtual void mulsd(RegXMM src, RegXMM srcdest) = 0;
841 virtual void subsd(RegXMM src, RegXMM srcdest) = 0;
842 virtual void pxor(RegXMM src, RegXMM srcdest) = 0;
843 virtual void cvtsi2sd(Reg64 src, RegXMM dest) = 0;
844 virtual void cvtsi2sd(MemoryRef m, RegXMM dest) = 0;
845 virtual void ucomisd(RegXMM l, RegXMM r) = 0;
846 virtual void sqrtsd(RegXMM src, RegXMM dest) = 0;
847 virtual void divsd(RegXMM src, RegXMM srcdest) = 0;
848 virtual void cvttsd2siq(RegXMM src, Reg64 dest) = 0;
849
850 static void patchJcc(CodeAddress jmp, CodeAddress from, CodeAddress dest) {
851 assert(jmp[0] == 0x0F && (jmp[1] & 0xF0) == 0x80);
852 ssize_t diff = dest - (from + 6);
853 *(int32_t*)(jmp + 2) = safe_cast<int32_t>(diff);
854 }
855
856 static void patchJcc8(CodeAddress jmp, CodeAddress from, CodeAddress dest) {
857 assert((jmp[0] & 0xF0) == 0x70);
858 ssize_t diff = dest - (from + 2); // one for opcode, one for offset
859 *(int8_t*)(jmp + 1) = safe_cast<int8_t>(diff);
860 }
861
862 static void patchJmp(CodeAddress jmp, CodeAddress from, CodeAddress dest) {
863 assert(jmp[0] == 0xE9);
864 ssize_t diff = dest - (from + 5);
865 *(int32_t*)(jmp + 1) = safe_cast<int32_t>(diff);
866 }
867
868 static void patchJmp8(CodeAddress jmp, CodeAddress from, CodeAddress dest) {
869 assert(jmp[0] == 0xEB);
870 ssize_t diff = dest - (from + 2); // one for opcode, one for offset
871 *(int8_t*)(jmp + 1) = safe_cast<int8_t>(diff);
872 }
873
874 static void patchCall(CodeAddress call, CodeAddress from, CodeAddress dest) {
875 assert(call[0] == 0xE8);
876 ssize_t diff = dest - (from + 5);
877 *(int32_t*)(call + 1) = safe_cast<int32_t>(diff);
878 }
879
880 virtual void emitInt3s(int n) = 0;
881 virtual void emitNop(int n) = 0;
882 virtual void pad() = 0;
883
884 void byte(uint8_t b) {
885 codeBlock.byte(b);
886 }
887 void word(uint16_t w) {
888 codeBlock.word(w);
889 }
890 void dword(uint32_t dw) {
891 codeBlock.dword(dw);
892 }
893 void qword(uint64_t qw) {
894 codeBlock.qword(qw);
895 }
896 void bytes(size_t n, const uint8_t* bs) {
897 codeBlock.bytes(n, bs);
898 }
899
900 virtual void cload_reg64_disp_reg64(ConditionCode cc, Reg64 rbase,
901 int off, Reg64 rdest) = 0;
902 virtual void cload_reg64_disp_reg32(ConditionCode cc, Reg64 rbase,
903 int off, Reg32 rdest) = 0;
904 virtual void cmov_reg64_reg64(ConditionCode cc, Reg64 rsrc,
905 Reg64 rdest) = 0;
906
907 protected:
908 RegNumber rn(Reg8 r) { return RegNumber(int(r)); }
909 RegNumber rn(Reg16 r) { return RegNumber(int(r)); }
910 RegNumber rn(Reg32 r) { return RegNumber(int(r)); }
911 RegNumber rn(Reg64 r) { return RegNumber(int(r)); }
912 RegNumber rn(RegXMM r) { return RegNumber(int(r)); }
913
914 CodeBlock& codeBlock;
915 };
916 }}
917
918 #ifdef HAVE_LIBXED
919 #include "hphp/util/asm-x64-intelxed.h"
920 #endif
921
922 #include "hphp/util/asm-x64-legacy.h"
923
924 namespace HPHP { namespace jit {
925 inline MemoryRef IndexedDispReg::operator*() const {
926 return MemoryRef(*this);
927 }
928
929 inline MemoryRef IndexedDispReg::operator[](intptr_t x) const {
930 return *(*this + x);
931 }
932
933 inline MemoryRef DispReg::operator*() const {
934 return MemoryRef(*this);
935 }
936
937 inline MemoryRef DispReg::operator[](intptr_t x) const {
938 return *(*this + x);
939 }
940
941 inline RIPRelativeRef DispRIP::operator*() const {
942 return RIPRelativeRef(*this);
943 }
944
945 inline RIPRelativeRef DispRIP::operator[](intptr_t x) const {
946 return *(*this + x);
947 }
948
949 inline DispReg operator+(Reg64 r, intptr_t d) { return DispReg(r, d); }
950 inline DispReg operator-(Reg64 r, intptr_t d) { return DispReg(r, -d); }
951 inline DispRIP operator+(RegRIP /*r*/, intptr_t d) {
952 return DispRIP(d);
953 }
954 inline DispRIP operator-(RegRIP /*r*/, intptr_t d) {
955 return DispRIP(d);
956 }
957
958 inline ScaledIndex operator*(Reg64 r, int scale) {
959 return ScaledIndex(r, scale);
960 }
961 inline IndexedDispReg operator+(Reg64 base, ScaledIndex sr) {
962 return IndexedDispReg(base, sr);
963 }
964 inline ScaledIndexDisp operator+(ScaledIndex si, intptr_t disp) {
965 return ScaledIndexDisp(si, disp);
966 }
967 inline IndexedDispReg operator+(Reg64 b, Reg64 i) {
968 return b + ScaledIndex(i, 0x1);
969 }
970
971 inline MemoryRef operator*(Reg64 r) { return MemoryRef(DispReg(r)); }
972 inline DispRIP operator*(RegRIP /*r*/) {
973 return DispRIP(0);
974 }
975
976 inline MemoryRef Reg64::operator[](intptr_t disp) const {
977 return *(*this + disp);
978 }
979
980 inline MemoryRef Reg64::operator[](Reg64 idx) const {
981 return *(*this + idx * 1);
982 }
983
984 inline MemoryRef Reg64::operator[](ScaledIndex si) const {
985 return *(*this + si);
986 }
987
988 inline MemoryRef Reg64::operator[](DispReg dr) const {
989 return *(*this + ScaledIndex(dr.base, 0x1) + dr.disp);
990 }
991
992 inline MemoryRef Reg64::operator[](ScaledIndexDisp sid) const {
993 return *(*this + sid.si + sid.disp);
994 }
995
996 inline RIPRelativeRef RegRIP::operator[](intptr_t disp) const {
997 return *(*this + disp);
998 }
999
1000 /*
1001 * Used for the x64 addressing mode where there is a displacement,
1002 * possibly with a scaled index, but no base register.
1003 */
1004 inline MemoryRef baseless(intptr_t disp) { return *(DispReg { disp }); }
1005 inline MemoryRef baseless(ScaledIndexDisp sid) {
1006 return *(IndexedDispReg { sid });
1007 }
1008
1009 //////////////////////////////////////////////////////////////////////
1010
1011 struct Label {
1012 explicit Label()
1013 : m_a(nullptr)
1014 , m_address(nullptr)
1015 {}
1016
1017 ~Label() {
1018 // Label had jumps but was never set -- this can happen if we fill the TC.
1019 if (!m_a || !m_address) {
1020 return;
1021 }
1022 for (auto& ji : m_toPatch) {
1023 auto realSrc = ji.a->toDestAddress(ji.addr);
1024 switch (ji.type) {
1025 case Branch::Jmp: ji.a->patchJmp(realSrc, ji.addr, m_address); break;
1026 case Branch::Jmp8: ji.a->patchJmp8(realSrc, ji.addr, m_address); break;
1027 case Branch::Jcc: ji.a->patchJcc(realSrc, ji.addr, m_address); break;
1028 case Branch::Jcc8: ji.a->patchJcc8(realSrc, ji.addr, m_address); break;
1029 case Branch::Call: ji.a->patchCall(realSrc, ji.addr, m_address); break;
1030 }
1031 }
1032 }
1033
1034 Label(const Label&) = delete;
1035 Label& operator=(const Label&) = delete;
1036
1037 void jmp(X64AssemblerBase& a) {
1038 addJump(&a, Branch::Jmp);
1039 a.jmp(m_address ? m_address : a.frontier());
1040 }
1041
1042 void jmp8(X64AssemblerBase& a) {
1043 addJump(&a, Branch::Jmp8);
1044 a.jmp8(m_address ? m_address : a.frontier());
1045 }
1046
1047 void jcc(X64AssemblerBase& a, ConditionCode cc) {
1048 addJump(&a, Branch::Jcc);
1049 a.jcc(cc, m_address ? m_address : a.frontier());
1050 }
1051
1052 void jcc8(X64AssemblerBase& a, ConditionCode cc) {
1053 addJump(&a, Branch::Jcc8);
1054 a.jcc8(cc, m_address ? m_address : a.frontier());
1055 }
1056
1057 void call(X64AssemblerBase& a) {
1058 addJump(&a, Branch::Call);
1059 a.call(m_address ? m_address : a.frontier());
1060 }
1061
1062 void jmpAuto(X64AssemblerBase& a) {
1063 assert(m_address);
1064 auto delta = m_address - (a.frontier() + 2);
1065 if (deltaFits(delta, sz::byte)) {
1066 jmp8(a);
1067 } else {
1068 jmp(a);
1069 }
1070 }
1071
1072 void jccAuto(X64AssemblerBase& a, ConditionCode cc) {
1073 assert(m_address);
1074 auto delta = m_address - (a.frontier() + 2);
1075 if (deltaFits(delta, sz::byte)) {
1076 jcc8(a, cc);
1077 } else {
1078 jcc(a, cc);
1079 }
1080 }
1081
1082 friend void asm_label(X64AssemblerBase& a, Label& l) {
1083 assert(!l.m_address && !l.m_a && "Label was already set");
1084 l.m_a = &a;
1085 l.m_address = a.frontier();
1086 }
1087
1088 private:
1089 enum class Branch {
1090 Jcc,
1091 Jcc8,
1092 Jmp,
1093 Jmp8,
1094 Call
1095 };
1096
1097 struct JumpInfo {
1098 Branch type;
1099 X64AssemblerBase* a;
1100 CodeAddress addr;
1101 };
1102
1103 private:
1104 void addJump(X64AssemblerBase* a, Branch type) {
1105 if (m_address) return;
1106 JumpInfo info;
1107 info.type = type;
1108 info.a = a;
1109 info.addr = a->codeBlock.frontier();
1110 m_toPatch.push_back(info);
1111 }
1112
1113 private:
1114 X64AssemblerBase* m_a;
1115 CodeAddress m_address;
1116 std::vector<JumpInfo> m_toPatch;
1117 };
1118
1119 inline void X64AssemblerBase::jmp(Label& l) { l.jmp(*this); }
1120 inline void X64AssemblerBase::jmp8(Label& l) { l.jmp8(*this); }
1121 inline void X64AssemblerBase::jcc(ConditionCode c, Label& l) {
1122 l.jcc(*this, c);
1123 }
1124 inline void X64AssemblerBase::jcc8(ConditionCode c, Label& l) {
1125 l.jcc8(*this, c);
1126 }
1127 inline void X64AssemblerBase::call(Label& l) { l.call(*this); }
1128
1129 #define CC(nm, code) \
1130 inline void X64AssemblerBase::j##nm(Label& l) { l.jcc(*this, code); } \
1131 inline void X64AssemblerBase::j##nm##8(Label& l) { l.jcc8(*this, code); }
1132 CCS
1133 #undef CC
1134
1135 //////////////////////////////////////////////////////////////////////
1136
1137 /*
1138 * Select the assembler which contains a given address.
1139 *
1140 * E.g.:
1141 *
1142 * Asm& a = codeBlockChoose(toPatch, a, acold);
1143 * a.patchJmp(...);
1144 */
1145 inline CodeBlock& codeBlockChoose(CodeAddress addr) {
1146 always_assert_flog(false,
1147 "address {} was not part of any known code block", addr);
1148 }
1149 template<class... Blocks>
1150 CodeBlock& codeBlockChoose(CodeAddress addr, CodeBlock& a, Blocks&... as) {
1151 if (a.contains(addr)) return a;
1152 return codeBlockChoose(addr, as...);
1153 }
1154
1155 //////////////////////////////////////////////////////////////////////
1156
1157 namespace x64 {
1158
1159 struct DecodedInstruction {
1160 DecodedInstruction(uint8_t* ip, uint8_t* base)
1161 : m_base(base)
1162 { decode(ip); }
1163
1164 explicit DecodedInstruction(uint8_t* ip) : DecodedInstruction(ip, ip) {}
1165
1166 std::string toString();
1167 size_t size() { return m_size; }
1168
1169 bool hasPicOffset() const { return m_flags.picOff; }
1170 uint8_t* picAddress() const;
1171 bool setPicAddress(uint8_t* target);
1172
1173 bool hasOffset() const { return m_offSz != 0; }
1174 int32_t offset() const;
1175
1176 bool hasImmediate() const { return m_immSz; }
1177 int64_t immediate() const;
1178 bool setImmediate(int64_t value);
1179 bool isNop() const;
1180 enum BranchType {
1181 Conditional = 1,
1182 Unconditional = 1 << 1,
1183 };
1184 bool isBranch(BranchType branchType = BranchType(Conditional |
1185 Unconditional)) const;
1186 bool isCall() const;
1187 bool isJmp() const;
1188 bool isLea() const;
1189 bool isFuseable(const DecodedInstruction& next) const;
1190 ConditionCode jccCondCode() const;
1191 bool shrinkBranch();
1192 void widenBranch();
1193 uint8_t getModRm() const;
1194 private:
1195 void decode(uint8_t* ip);
1196 bool decodePrefix(uint8_t* ip);
1197 int decodeRexVexXop(uint8_t* ip);
1198 int decodeOpcode(uint8_t* ip);
1199 void determineOperandsMap0(uint8_t* ip);
1200 void determineOperandsMap1(uint8_t* ip);
1201 void determineOperandsMap2(uint8_t* ip);
1202 void determineOperandsMap3(uint8_t* ip);
1203 int decodeModRm(uint8_t* ip);
1204 int decodeImm(uint8_t* ip);
1205
1206 // We may wish to decode an instruction whose address is m_ip, but treat all
1207 // PIC references as relative to m_base.
1208 uint8_t* m_base;
1209
1210 uint8_t* m_ip;
1211 uint32_t m_size;
1212
1213 union {
1214 uint32_t m_flagsVal;
1215 struct {
1216 uint32_t lock : 1;
1217 uint32_t repNE : 1;
1218 uint32_t rep : 1;
1219
1220 uint32_t cs : 1;
1221 uint32_t ss : 1;
1222 uint32_t ds : 1;
1223 uint32_t es : 1;
1224 uint32_t fs : 1;
1225 uint32_t gs : 1;
1226 uint32_t bTaken : 1;
1227 uint32_t bNotTaken : 1;
1228
1229 uint32_t opndSzOvr : 1;
1230 uint32_t addrSzOvr : 1;
1231
1232 uint32_t rex : 1;
1233 uint32_t vex : 1;
1234 uint32_t xop : 1;
1235
1236 uint32_t w : 1;
1237 uint32_t r : 1;
1238 uint32_t x : 1;
1239 uint32_t b : 1;
1240 uint32_t l : 1;
1241
1242 uint32_t def64 : 1;
1243 uint32_t immIsAddr : 1;
1244 uint32_t picOff : 1;
1245 uint32_t hasModRm : 1;
1246 uint32_t hasSib : 1;
1247 } m_flags;
1248 };
1249
1250 uint8_t m_map_select;
1251 uint8_t m_xtra_op;
1252 uint8_t m_opcode;
1253 uint8_t m_immSz;
1254 uint8_t m_offSz;
1255 };
1256
1257 constexpr DecodedInstruction::BranchType operator|(
1258 DecodedInstruction::BranchType a,
1259 DecodedInstruction::BranchType b
1260 ) {
1261 return DecodedInstruction::BranchType((int)a | (int)b);
1262 }
1263
1264 inline DecodedInstruction::BranchType& operator|=(
1265 DecodedInstruction::BranchType& a,
1266 const DecodedInstruction::BranchType& b
1267 ) {
1268 return (a = DecodedInstruction::BranchType((int)a | (int)b));
1269 }
1270
1271 #undef TRACEMOD
1272 #undef logical_const
1273 #undef CCS
1274
1275 }}}