hphp/runtime/vm/jit/vasm-x64.h

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #ifndef incl_HPHP_JIT_VASM_X64_H_
  18 #define incl_HPHP_JIT_VASM_X64_H_
  19
  20 #include "hphp/runtime/base/stats.h"
  21 #include "hphp/runtime/base/types.h"
  22 #include "hphp/runtime/vm/jit/abi.h"
  23 #include "hphp/runtime/vm/jit/containers.h"
  24 #include "hphp/runtime/vm/jit/cpp-call.h"
  25 #include "hphp/runtime/vm/jit/fixup.h"
  26 #include "hphp/runtime/vm/jit/phys-reg.h"
  27 #include "hphp/runtime/vm/jit/service-requests.h"
  28 #include "hphp/runtime/vm/jit/target-cache.h"
  29 #include "hphp/runtime/vm/jit/vasm.h"
  30 #include "hphp/runtime/vm/jit/vasm-reg.h"
  31 #include "hphp/runtime/vm/srckey.h"
  32 #include "hphp/util/asm-x64.h"
  33
  34 #include <bitset>
  35 #include <boost/dynamic_bitset.hpp>
  36 #include <folly/Range.h>
  37
  38 namespace HPHP { namespace jit {
  39 struct IRInstruction;
  40 struct AsmInfo;
  41 }}
  42
  43 namespace HPHP { namespace jit {
  44
  45 // XXX: This should go back to arg-group.h as part of work on t5297892
  46 enum class DestType : uint8_t {
  47   None,  // return void (no valid registers)
  48   SSA,   // return a single-register value
  49   Byte,  // return a single-byte register value
  50   TV,    // return a TypedValue packed in two registers
  51   Dbl,   // return scalar double in a single FP register
  52   SIMD,  // return a TypedValue in one SIMD register
  53 };
  54 const char* destTypeName(DestType);
  55
  56 // instantiations of this wrap virtual register numbers in in a strongly
  57 // typed wrapper that conveys physical constraints, similar to Reg64,
  58 // Reg32, RegXMM, etc.
  59 template<class Reg, VregKind Kind, int Bits> struct Vr {
  60   static constexpr auto bits = Bits;
  61   static constexpr auto kind = Kind;
  62   explicit Vr(size_t rn) : rn(rn) {}
  63   /* implicit */ Vr(Vreg r) : rn(size_t(r)) {
  64     if (kind == VregKind::Gpr) {
  65       assert(!r.isValid() || r.isVirt() || r.isGP());
  66     } else if (kind == VregKind::Simd) {
  67       assert(!r.isValid() || r.isVirt() || r.isSIMD());
  68     } else if (kind == VregKind::Sf) {
  69       assert(!r.isValid() || r.isVirt() || r.isSF());
  70     }
  71   }
  72   /* implicit */ Vr(Reg r) : Vr{Vreg(r)} {}
  73   /* implicit */ Vr(PhysReg pr) : Vr{Vreg(pr)} {}
  74   Reg asReg() const {
  75     assert(isPhys());
  76     return isGP() ? Reg(rn) :
  77            isSIMD() ? Reg(rn-Vreg::X0) :
  78            /* isSF() ? */ Reg(rn-Vreg::S0);
  79   }
  80   /* implicit */ operator Reg() const { return asReg(); }
  81   /* implicit */ operator Vreg() const { return Vreg(rn); }
  82   /* implicit */ operator size_t() const { return rn; }
  83   bool isPhys() const {
  84     static_assert(Vreg::G0 == 0, "");
  85     return rn < Vreg::V0;
  86   }
  87   bool isGP() const { return rn>=Vreg::G0 && rn<Vreg::G0+Vreg::kNumGP; }
  88   bool isSIMD() const { return rn>=Vreg::X0 && rn<Vreg::X0+Vreg::kNumXMM; }
  89   bool isSF() const { return rn>=Vreg::S0 && rn<Vreg::S0+Vreg::kNumSF; }
  90   bool isVirt() const { return rn >= Vreg::V0 && isValid(); }
  91   bool operator==(Vr<Reg,Kind,Bits> r) const { return rn == r.rn; }
  92   bool operator!=(Vr<Reg,Kind,Bits> r) const { return rn != r.rn; }
  93   bool isValid() const { return rn != Vreg::kInvalidReg; }
  94   Vptr operator[](int disp) const;
  95   Vptr operator[](ScaledIndex si) const;
  96   Vptr operator[](ScaledIndexDisp sid) const;
  97   Vptr operator[](Vptr) const;
  98   Vptr operator[](DispReg) const;
  99   Vptr operator*() const;
 100   Vptr operator+(size_t d) const;
 101   explicit operator PhysReg() const { return asReg(); }
 102 private:
 103   unsigned rn;
 104 };
 105 typedef Vr<Reg64,VregKind::Gpr,64>    Vreg64;
 106 typedef Vr<Reg32,VregKind::Gpr,32>    Vreg32;
 107 typedef Vr<Reg16,VregKind::Gpr,16>    Vreg16;
 108 typedef Vr<Reg8,VregKind::Gpr,8>      Vreg8;
 109 typedef Vr<RegXMM,VregKind::Simd,64>  VregDbl;
 110 typedef Vr<RegXMM,VregKind::Simd,128> Vreg128;
 111 typedef Vr<RegSF,VregKind::Sf,4>      VregSF;
 112
 113 inline Reg64 r64(Vreg64 r) { return r; }
 114
 115 // base + index*scale + disp.
 116 // base is optional, implying baseless address
 117 // index is optional
 118 struct Vptr {
 119   enum Segment: uint8_t { DS, FS };
 120   template<class Base> Vptr(Base b, int d)
 121     : base(b), index(0xffffffff), scale(1), disp(d)
 122   {}
 123   template<class Base, class Index> Vptr(Base b, Index i, int s, int d)
 124     : base(b), index(i), scale(s), disp(d)
 125   {}
 126   /* implicit */ Vptr(MemoryRef m, Segment s = DS)
 127     : base(m.r.base), index(m.r.index), scale(m.r.scale)
 128     , seg(s), disp(m.r.disp)
 129   {}
 130   MemoryRef mr() const {
 131     if (index.isValid()) {
 132       return base.isValid() ? r64(base)[r64(index) * scale + disp] :
 133              *(IndexedDispReg{r64(index) * scale + disp});
 134     } else {
 135       return base.isValid() ? r64(base)[disp] :
 136              *(DispReg{disp});
 137     }
 138   }
 139   /* implicit */ operator MemoryRef() const {
 140     assert(seg == DS);
 141     return mr();
 142   }
 143
 144   Vreg64 base; // optional, for baseless mode
 145   Vreg64 index; // optional
 146   uint8_t scale; // 1,2,4,8
 147   Segment seg{DS}; // DS or FS
 148   int32_t disp;
 149 };
 150
 151 struct Vscaled {
 152   Vreg64 index;
 153   int scale;
 154 };
 155
 156 inline Vptr operator+(Vptr lhs, int32_t d) {
 157   return Vptr(lhs.base, lhs.index, lhs.scale, lhs.disp + d);
 158 }
 159
 160 inline Vptr operator+(Vptr lhs, intptr_t d) {
 161   return Vptr(lhs.base, lhs.index, lhs.scale,
 162               safe_cast<int32_t>(lhs.disp + d));
 163 }
 164
 165 inline Vptr operator+(Vreg64 base, int32_t d) {
 166   return Vptr(base, d);
 167 }
 168
 169 // A Vloc is either a single or pair of vregs, for keeping track
 170 // of where we have stored an SSATmp.
 171 struct Vloc {
 172   enum Kind { kPair, kWide };
 173   Vloc() {}
 174   explicit Vloc(Vreg r) { m_regs[0] = r; }
 175   Vloc(Vreg r0, Vreg r1) { m_regs[0] = r0; m_regs[1] = r1; }
 176   Vloc(Kind kind, Vreg r) : m_kind(kind) { m_regs[0] = r; }
 177   bool hasReg(int i = 0) const {
 178     return m_regs[i].isValid();
 179   }
 180   Vreg reg(int i = 0) const {
 181     return m_regs[i];
 182   }
 183   int numAllocated() const {
 184     return int(m_regs[0].isValid()) + int(m_regs[1].isValid());
 185   }
 186   int numWords() const {
 187     return m_kind == kWide ? 2 : numAllocated();
 188   }
 189   bool isFullSIMD() const {
 190     return m_kind == kWide;
 191   }
 192
 193   bool operator==(Vloc other) const {
 194     return m_kind == other.m_kind &&
 195            m_regs[0] == other.m_regs[0] &&
 196            m_regs[1] == other.m_regs[1];
 197   }
 198   bool operator!=(Vloc other) const {
 199     return !(*this == other);
 200   }
 201
 202 private:
 203   Kind m_kind{kPair};
 204   Vreg m_regs[2];
 205 };
 206
 207 inline Vscaled Vreg::operator*(int scale) const {
 208   return Vscaled{*this, scale};
 209 }
 210
 211 inline Vptr Vreg::operator[](Vscaled si) const {
 212   return Vptr(*this, si.index, si.scale, 0);
 213 }
 214
 215 inline Vptr Vreg::operator*() const { return Vptr(*this, 0); }
 216 inline Vptr Vreg::operator[](int disp) const { return Vptr(*this, disp); }
 217 inline Vptr Vreg::operator[](ScaledIndex si) const {
 218   return Vptr(*this, si.index, si.scale, 0);
 219 }
 220 inline Vptr Vreg::operator[](ScaledIndexDisp sid) const {
 221   return Vptr(*this, sid.si.index, sid.si.scale, sid.disp);
 222 }
 223 inline Vptr Vreg::operator[](Vptr p) const {
 224   return Vptr(*this, p.base, 1, p.disp);
 225 }
 226 inline Vptr Vreg::operator[](DispReg rd) const {
 227   return Vptr(*this, rd.base, 1, rd.disp);
 228 }
 229 inline Vptr Vreg::operator[](Vreg index) const {
 230   return Vptr(*this, index, 1, 0);
 231 }
 232
 233 template<class Reg, VregKind Kind, int Bits>
 234 Vptr Vr<Reg,Kind,Bits>::operator*() const {
 235   return Vptr(*this, 0);
 236 }
 237
 238 template<class Reg, VregKind Kind, int Bits>
 239 Vptr Vr<Reg,Kind,Bits>::operator[](int disp) const {
 240   return Vptr(*this, disp);
 241 }
 242
 243 inline Vptr operator+(Vreg base, int32_t d) {
 244   return Vptr(base, safe_cast<int32_t>(d));
 245 }
 246
 247 inline Vptr operator+(Vreg base, intptr_t d) {
 248   return Vptr(base, safe_cast<int32_t>(d));
 249 }
 250
 251 inline Vptr Vreg::operator+(size_t d) const {
 252   return Vptr(*this, safe_cast<int32_t>(d));
 253 }
 254
 255 inline Vptr operator+(Vreg64 base, intptr_t d) {
 256   return Vptr(base, safe_cast<int32_t>(d));
 257 }
 258
 259 template<class Reg, VregKind Kind, int Bits>
 260 Vptr Vr<Reg,Kind,Bits>::operator[](ScaledIndex si) const {
 261   return Vptr(*this, si.index, si.scale, 0);
 262 }
 263
 264 template<class Reg, VregKind Kind, int Bits>
 265 Vptr Vr<Reg,Kind,Bits>::operator[](ScaledIndexDisp sid) const {
 266   return Vptr(*this, sid.si.index, sid.si.scale, sid.disp);
 267 }
 268
 269 template<class Reg, VregKind Kind, int Bits>
 270 Vptr Vr<Reg,Kind,Bits>::operator[](Vptr p) const {
 271   return Vptr(*this, p.base, 1, p.disp);
 272 }
 273
 274 template<class Reg, VregKind Kind, int Bits>
 275 Vptr Vr<Reg,Kind,Bits>::operator[](DispReg rd) const {
 276   return Vptr(*this, rd.base, 1, rd.disp);
 277 }
 278
 279 template<class Reg, VregKind Kind, int Bits>
 280 inline Vptr Vr<Reg,Kind,Bits>::operator+(size_t d) const {
 281   return Vptr(*this, safe_cast<int32_t>(d));
 282 }
 283
 284 // Field actions:
 285 // I(f)     immediate
 286 // Inone    no immediates
 287 // U(s)     use s
 288 // UA(s)    use s, but s lifetime extends across the instruction
 289 // UH(s,h)  use s, try assigning same register as h
 290 // D(d)     define d
 291 // DH(d,h)  define d, try assigning same register as h
 292 // Un,Dn    no uses, defs
 293
 294 #define VASM_OPCODES\
 295   /* service requests, PHP-level function calls */\
 296   O(bindaddr, I(dest) I(sk), Un, Dn)\
 297   O(bindcall, I(stub), U(args), Dn)\
 298   O(bindexit, I(cc) I(target), U(sf) U(args), Dn)\
 299   O(bindjcc1st, I(cc) I(targets[0]) I(targets[1]), U(sf) U(args), Dn)\
 300   O(bindjcc2nd, I(cc) I(target), U(sf) U(args), Dn)\
 301   O(bindjmp, I(target) I(trflags), U(args), Dn)\
 302   O(callstub, I(target) I(kills) I(fix), U(args), Dn)\
 303   O(contenter, Inone, U(fp) U(target) U(args), Dn)\
 304   /* vasm intrinsics */\
 305   O(copy, Inone, UH(s,d), DH(d,s))\
 306   O(copy2, Inone, UH(s0,d0) UH(s1,d1), DH(d0,s0) DH(d1,s1))\
 307   O(copyargs, Inone, UH(s,d), DH(d,s))\
 308   O(debugtrap, Inone, Un, Dn)\
 309   O(fallthru, Inone, Un, Dn)\
 310   O(ldimmb, I(s) I(saveflags), Un, D(d))\
 311   O(ldimm, I(s) I(saveflags), Un, D(d))\
 312   O(fallback, I(dest), U(args), Dn)\
 313   O(fallbackcc, I(cc) I(dest), U(sf) U(args), Dn)\
 314   O(kpcall, I(target) I(callee) I(prologIndex), U(args), Dn)\
 315   O(ldpoint, I(s), Un, D(d))\
 316   O(load, Inone, U(s), D(d))\
 317   O(mccall, I(target), U(args), Dn)\
 318   O(mcprep, Inone, Un, D(d))\
 319   O(nothrow, Inone, Un, Dn)\
 320   O(phidef, Inone, Un, D(defs))\
 321   O(phijmp, Inone, U(uses), Dn)\
 322   O(phijcc, I(cc), U(uses) U(sf), Dn)\
 323   O(point, I(p), Un, Dn)\
 324   O(store, Inone, U(s) U(d), Dn)\
 325   O(svcreq, I(req) I(stub_block), U(args), Dn)\
 326   O(syncpoint, I(fix), Un, Dn)\
 327   O(unwind, Inone, Un, Dn)\
 328   O(vcall, I(call) I(destType) I(fixup), U(args), D(d))\
 329   O(vinvoke, I(call) I(destType) I(fixup), U(args), D(d))\
 330   O(landingpad, Inone, Un, Dn)\
 331   O(defvmsp, Inone, Un, D(d))\
 332   O(syncvmsp, Inone, U(s), Dn)\
 333   O(syncvmfp, Inone, U(s), Dn)\
 334   O(srem, Inone, U(s0) U(s1), D(d))\
 335   O(sar, Inone, U(s0) U(s1), D(d) D(sf))\
 336   O(shl, Inone, U(s0) U(s1), D(d) D(sf))\
 337   O(ldretaddr, Inone, U(s), D(d))\
 338   O(movretaddr, Inone, U(s), D(d))\
 339   O(retctrl, Inone, U(s), Dn)\
 340   O(absdbl, Inone, U(s), D(d))\
 341   /* arm instructions */\
 342   O(asrv, Inone, U(sl) U(sr), D(d))\
 343   O(brk, I(code), Un, Dn)\
 344   O(cbcc, I(cc), U(s), Dn)\
 345   O(hcsync, I(fix) I(call), Un, Dn)\
 346   O(hcnocatch, I(call), Un, Dn)\
 347   O(hcunwind, I(call), Un, Dn)\
 348   O(hostcall, I(argc) I(syncpoint), U(args), Dn)\
 349   O(lslv, Inone, U(sl) U(sr), D(d))\
 350   O(tbcc, I(cc) I(bit), U(s), Dn)\
 351   /* x64 instructions */\
 352   O(addli, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 353   O(addlm, Inone, U(s0) U(m), D(sf)) \
 354   O(addq, Inone, U(s0) U(s1), D(d) D(sf)) \
 355   O(addqi, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 356   O(addqim, I(s0), U(m), D(sf)) \
 357   O(addsd, Inone, U(s0) U(s1), D(d))\
 358   O(andb, Inone, U(s0) U(s1), D(d) D(sf)) \
 359   O(andbi, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 360   O(andbim, I(s), U(m), D(sf)) \
 361   O(andl, Inone, U(s0) U(s1), D(d) D(sf)) \
 362   O(andli, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 363   O(andq, Inone, U(s0) U(s1), D(d) D(sf)) \
 364   O(andqi, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 365   O(call, I(target), U(args), Dn)\
 366   O(callm, Inone, U(target) U(args), Dn)\
 367   O(callr, Inone, U(target) U(args), Dn)\
 368   O(cloadq, I(cc), U(sf) U(f) U(t), D(d))\
 369   O(cmovq, I(cc), U(sf) U(f) U(t), D(d))\
 370   O(cmpb, Inone, U(s0) U(s1), D(sf))\
 371   O(cmpbi, I(s0), U(s1), D(sf))\
 372   O(cmpbim, I(s0), U(s1), D(sf))\
 373   O(cmpl, Inone, U(s0) U(s1), D(sf))\
 374   O(cmpli, I(s0), U(s1), D(sf))\
 375   O(cmplim, I(s0), U(s1), D(sf))\
 376   O(cmplm, Inone, U(s0) U(s1), D(sf))\
 377   O(cmpq, Inone, U(s0) U(s1), D(sf))\
 378   O(cmpqi, I(s0), U(s1), D(sf))\
 379   O(cmpqim, I(s0), U(s1), D(sf))\
 380   O(cmpqm, Inone, U(s0) U(s1), D(sf))\
 381   O(cmpsd, I(pred), UA(s0) U(s1), D(d))\
 382   O(cqo, Inone, Un, Dn)\
 383   O(cvttsd2siq, Inone, U(s), D(d))\
 384   O(cvtsi2sd, Inone, U(s), D(d))\
 385   O(cvtsi2sdm, Inone, U(s), D(d))\
 386   O(decl, Inone, UH(s,d), DH(d,s) D(sf))\
 387   O(declm, Inone, U(m), D(sf))\
 388   O(decq, Inone, UH(s,d), DH(d,s) D(sf))\
 389   O(decqm, Inone, U(m), D(sf))\
 390   O(divsd, Inone, UA(s0) U(s1), D(d))\
 391   O(idiv, Inone, U(s), D(sf))\
 392   O(imul, Inone, U(s0) U(s1), D(d) D(sf))\
 393   O(incwm, Inone, U(m), D(sf))\
 394   O(incl, Inone, UH(s,d), DH(d,s) D(sf))\
 395   O(inclm, Inone, U(m), D(sf))\
 396   O(incq, Inone, UH(s,d), DH(d,s) D(sf))\
 397   O(incqm, Inone, U(m), D(sf))\
 398   O(incqmlock, Inone, U(m), D(sf))\
 399   O(jcc, I(cc), U(sf), Dn)\
 400   O(jmp, Inone, Un, Dn)\
 401   O(jmpr, Inone, U(target) U(args), Dn)\
 402   O(jmpm, Inone, U(target) U(args), Dn)\
 403   O(lea, Inone, U(s), D(d))\
 404   O(leap, I(s), Un, D(d))\
 405   O(loaddqu, Inone, U(s), D(d))\
 406   O(loadb, Inone, U(s), D(d))\
 407   O(loadl, Inone, U(s), D(d))\
 408   O(loadqp, I(s), Un, D(d))\
 409   O(loadsd, Inone, U(s), D(d))\
 410   O(loadzbl, Inone, U(s), D(d))\
 411   O(loadzbq, Inone, U(s), D(d))\
 412   O(loadzlq, Inone, U(s), D(d))\
 413   O(movb, Inone, UH(s,d), DH(d,s))\
 414   O(movl, Inone, UH(s,d), DH(d,s))\
 415   O(movzbl, Inone, UH(s,d), DH(d,s))\
 416   O(movzbq, Inone, UH(s,d), DH(d,s))\
 417   O(mulsd, Inone, U(s0) U(s1), D(d))\
 418   O(mul, Inone, U(s0) U(s1), D(d))\
 419   O(neg, Inone, UH(s,d), DH(d,s) D(sf))\
 420   O(nop, Inone, Un, Dn)\
 421   O(not, Inone, UH(s,d), DH(d,s))\
 422   O(orq, Inone, U(s0) U(s1), D(d) D(sf))\
 423   O(orqi, I(s0), UH(s1,d), DH(d,s1) D(sf)) \
 424   O(orqim, I(s0), U(m), D(sf))\
 425   O(pop, Inone, Un, D(d))\
 426   O(popm, Inone, U(m), Dn)\
 427   O(psllq, I(s0), UH(s1,d), DH(d,s1))\
 428   O(psrlq, I(s0), UH(s1,d), DH(d,s1))\
 429   O(push, Inone, U(s), Dn)\
 430   O(pushm, Inone, U(s), Dn)\
 431   O(ret, Inone, U(args), Dn)\
 432   O(roundsd, I(dir), U(s), D(d))\
 433   O(sarq, Inone, UH(s,d), DH(d,s) D(sf))\
 434   O(sarqi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 435   O(setcc, I(cc), U(sf), D(d))\
 436   O(shlli, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 437   O(shlq, Inone, UH(s,d), DH(d,s) D(sf))\
 438   O(shlqi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 439   O(shrli, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 440   O(shrqi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 441   O(sqrtsd, Inone, U(s), D(d))\
 442   O(storeb, Inone, U(s) U(m), Dn)\
 443   O(storebi, I(s), U(m), Dn)\
 444   O(storedqu, Inone, U(s) U(m), Dn)\
 445   O(storel, Inone, U(s) U(m), Dn)\
 446   O(storeli, I(s), U(m), Dn)\
 447   O(storeqi, I(s), U(m), Dn)\
 448   O(storesd, Inone, U(s) U(m), Dn)\
 449   O(storew, Inone, U(s) U(m), Dn)\
 450   O(storewi, I(s), U(m), Dn)\
 451   O(subbi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 452   O(subl, Inone, UA(s0) U(s1), D(d) D(sf))\
 453   O(subli, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 454   O(subq, Inone, UA(s0) U(s1), D(d) D(sf))\
 455   O(subqi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 456   O(subsd, Inone, UA(s0) U(s1), D(d))\
 457   O(testb, Inone, U(s0) U(s1), D(sf))\
 458   O(testbi, I(s0), U(s1), D(sf))\
 459   O(testbim, I(s0), U(s1), D(sf))\
 460   O(testl, Inone, U(s0) U(s1), D(sf))\
 461   O(testli, I(s0), U(s1), D(sf))\
 462   O(testlim, I(s0), U(s1), D(sf))\
 463   O(testq, Inone, U(s0) U(s1), D(sf))\
 464   O(testqm, Inone, U(s0) U(s1), D(sf))\
 465   O(testqim, I(s0), U(s1), D(sf))\
 466   O(ucomisd, Inone, U(s0) U(s1), D(sf))\
 467   O(ud2, Inone, Un, Dn)\
 468   O(unpcklpd, Inone, UA(s0) U(s1), D(d))\
 469   O(xorb, Inone, U(s0) U(s1), D(d) D(sf))\
 470   O(xorbi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 471   O(xorq, Inone, U(s0) U(s1), D(d) D(sf))\
 472   O(xorqi, I(s0), UH(s1,d), DH(d,s1) D(sf))\
 473
 474 // intrinsics
 475 struct bindaddr { TCA* dest; SrcKey sk; };
 476 struct bindcall { TCA stub; RegSet args; };
 477 struct bindexit { ConditionCode cc; VregSF sf; SrcKey target;
 478                   TransFlags trflags; RegSet args; };
 479 struct bindjcc1st { ConditionCode cc; VregSF sf; Offset targets[2];
 480                     RegSet args; };
 481 struct bindjcc2nd { ConditionCode cc; VregSF sf; Offset target; RegSet args; };
 482 struct bindjmp { SrcKey target; TransFlags trflags; RegSet args; };
 483 struct callstub { CodeAddress target; RegSet args, kills; Fixup fix; };
 484 struct contenter { Vreg64 fp, target; RegSet args; };
 485 struct vcall { CppCall call; VcallArgsId args; Vtuple d;
 486                Fixup fixup; DestType destType; bool nothrow; };
 487 struct vinvoke { CppCall call; VcallArgsId args; Vtuple d; Vlabel targets[2];
 488                  Fixup fixup; DestType destType; bool smashable; };
 489 struct copy { Vreg s, d; };
 490 struct copy2 { Vreg64 s0, s1, d0, d1; };
 491 struct copyargs { Vtuple s, d; };
 492 struct debugtrap {};
 493 struct ldretaddr { Vptr s; Vreg d; };
 494 struct movretaddr { Vreg s, d; };
 495 struct retctrl { Vreg s; };
 496 struct absdbl { Vreg s, d; };
 497
 498 // No-op, used for marking the end of a block that is intentionally going to
 499 // fall-through.  Only for use with Vauto.
 500 struct fallthru {};
 501
 502 struct ldimmb { Immed s; Vreg8 d; bool saveflags; };
 503 struct ldimm  { Immed64 s; Vreg d; bool saveflags; };
 504 struct fallback { SrcKey dest; TransFlags trflags; RegSet args; };
 505 struct fallbackcc { ConditionCode cc; VregSF sf; SrcKey dest;
 506                     TransFlags trflags; RegSet args; };
 507 struct kpcall { CodeAddress target; const Func* callee; unsigned prologIndex;
 508                 RegSet args; };
 509 struct ldpoint { Vpoint s; Vreg64 d; };
 510 struct load { Vptr s; Vreg d; };
 511 struct mccall { CodeAddress target; RegSet args; };
 512 struct mcprep { Vreg64 d; };
 513 struct nothrow {};
 514 struct phidef { Vtuple defs; };
 515 struct phijmp { Vlabel target; Vtuple uses; };
 516 struct phijcc { ConditionCode cc; VregSF sf; Vlabel targets[2]; Vtuple uses; };
 517 struct point { Vpoint p; };
 518 struct store { Vreg s; Vptr d; };
 519 struct svcreq { ServiceRequest req; Vtuple args; TCA stub_block; };
 520 struct syncpoint { Fixup fix; };
 521 struct unwind { Vlabel targets[2]; };
 522 struct landingpad {};
 523 struct defvmsp { Vreg d; };
 524 struct syncvmsp { Vreg s; };
 525 struct syncvmfp { Vreg s; };
 526 struct srem { Vreg s0, s1, d; };
 527 struct sar { Vreg s0, s1, d; VregSF sf; };
 528 struct shl { Vreg s0, s1, d; VregSF sf; };
 529
 530 // arm-specific intrinsics
 531 struct hcsync { Fixup fix; Vpoint call; };
 532 struct hcnocatch { Vpoint call; };
 533 struct hcunwind { Vpoint call; Vlabel targets[2]; };
 534
 535 // arm specific instructions
 536 struct brk { uint16_t code; };
 537 struct hostcall { RegSet args; uint8_t argc; Vpoint syncpoint; };
 538 struct cbcc { vixl::Condition cc; Vreg64 s; Vlabel targets[2]; };
 539 struct tbcc { vixl::Condition cc; unsigned bit; Vreg64 s; Vlabel targets[2]; };
 540 struct lslv { Vreg64 sl, sr, d; };
 541 struct asrv { Vreg64 sl, sr, d; };
 542 struct mul { Vreg64 s0, s1, d; };
 543
 544 // ATT style operand order. for binary ops:
 545 // op   s0 s1 d:  d = s1 op s0    =>   d=s1; d op= s0
 546 // op   imm s1 d: d = s1 op imm   =>   d=s1; d op= imm
 547 // cmp  s0 s1:    s1 cmp s0
 548
 549 // suffix conventions:
 550 //  b   8-bit
 551 //  w   16-bit
 552 //  l   32-bit
 553 //  q   64-bit
 554 //  i   immediate
 555 //  m   Vptr
 556 //  p   RIPRelativeRef
 557
 558 // x64 instructions
 559 struct addli { Immed s0; Vreg32 s1, d; VregSF sf; };
 560 struct addlm { Vreg32 s0; Vptr m; VregSF sf; };
 561 struct addq  { Vreg64 s0, s1, d; VregSF sf; };
 562 struct addqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 563 struct addqim { Immed s0; Vptr m; VregSF sf; };
 564 struct addsd  { VregDbl s0, s1, d; };
 565 struct andb  { Vreg8 s0, s1, d; VregSF sf; };
 566 struct andbi { Immed s0; Vreg8 s1, d; VregSF sf; };
 567 struct andbim { Immed s; Vptr m; VregSF sf; };
 568 struct andl  { Vreg32 s0, s1, d; VregSF sf; };
 569 struct andli { Immed s0; Vreg32 s1, d; VregSF sf; };
 570 struct andq  { Vreg64 s0, s1, d; VregSF sf; };
 571 struct andqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 572 struct call { CodeAddress target; RegSet args; };
 573 struct callm { Vptr target; RegSet args; };
 574 struct callr { Vreg64 target; RegSet args; };
 575 struct cloadq { ConditionCode cc; VregSF sf; Vreg64 f; Vptr t; Vreg64 d; };
 576 struct cmovq { ConditionCode cc; VregSF sf; Vreg64 f, t, d; };
 577 struct cmpb  { Vreg8  s0; Vreg8  s1; VregSF sf; };
 578 struct cmpbi { Immed  s0; Vreg8  s1; VregSF sf; };
 579 struct cmpbim { Immed s0; Vptr s1; VregSF sf; };
 580 struct cmpl  { Vreg32 s0; Vreg32 s1; VregSF sf; };
 581 struct cmpli { Immed  s0; Vreg32 s1; VregSF sf; };
 582 struct cmplim { Immed s0; Vptr s1; VregSF sf; };
 583 struct cmplm { Vreg32 s0; Vptr s1; VregSF sf; };
 584 struct cmpq  { Vreg64 s0; Vreg64 s1; VregSF sf; };
 585 struct cmpqi { Immed  s0; Vreg64 s1; VregSF sf; };
 586 struct cmpqim { Immed s0; Vptr s1; VregSF sf; };
 587 struct cmpqm { Vreg64 s0; Vptr s1; VregSF sf; };
 588 struct cmpsd { ComparisonPred pred; VregDbl s0, s1, d; };
 589 struct cqo {};
 590 struct cvttsd2siq { VregDbl s; Vreg64 d; };
 591 struct cvtsi2sd { Vreg64 s; VregDbl d; };
 592 struct cvtsi2sdm { Vptr s; VregDbl d; };
 593 struct decl { Vreg32 s, d; VregSF sf; };
 594 struct declm { Vptr m; VregSF sf; };
 595 struct decq { Vreg64 s, d; VregSF sf; };
 596 struct decqm { Vptr m; VregSF sf; };
 597 struct divsd { VregDbl s0, s1, d; };
 598 struct idiv { Vreg64 s; VregSF sf; };
 599 struct imul { Vreg64 s0, s1, d; VregSF sf; };
 600 struct incl { Vreg32 s, d; VregSF sf; };
 601 struct inclm { Vptr m; VregSF sf; };
 602 struct incq { Vreg64 s, d; VregSF sf; };
 603 struct incqm { Vptr m; VregSF sf; };
 604 struct incqmlock { Vptr m; VregSF sf; };
 605 struct incwm { Vptr m; VregSF sf; };
 606 struct jcc { ConditionCode cc; VregSF sf; Vlabel targets[2]; };
 607 struct jmp { Vlabel target; };
 608 struct jmpr { Vreg64 target; RegSet args; };
 609 struct jmpm { Vptr target; RegSet args; };
 610 struct lea { Vptr s; Vreg64 d; };
 611 struct leap { RIPRelativeRef s; Vreg64 d; };
 612 struct loaddqu { Vptr s; Vreg128 d; };
 613 struct loadb { Vptr s; Vreg8 d; };
 614 struct loadl { Vptr s; Vreg32 d; };
 615 struct loadqp { RIPRelativeRef s; Vreg64 d; };
 616 struct loadsd { Vptr s; VregDbl d; };
 617 struct loadzbl { Vptr s; Vreg32 d; };
 618 struct loadzbq { Vptr s; Vreg64 d; };
 619 struct loadzlq { Vptr s; Vreg64 d; };
 620 struct movb { Vreg8 s, d; };
 621 struct movl { Vreg32 s, d; };
 622 struct movzbl { Vreg8 s; Vreg32 d; };
 623 struct movzbq { Vreg8 s; Vreg64 d; };
 624 struct mulsd  { VregDbl s0, s1, d; };
 625 struct neg { Vreg64 s, d; VregSF sf; };
 626 struct nop {};
 627 struct not { Vreg64 s, d; };
 628 struct orq { Vreg64 s0, s1, d; VregSF sf; };
 629 struct orqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 630 struct orqim { Immed s0; Vptr m; VregSF sf; };
 631 struct pop { Vreg64 d; };
 632 struct popm { Vptr m; };
 633 struct psllq { Immed s0; VregDbl s1, d; };
 634 struct psrlq { Immed s0; VregDbl s1, d; };
 635 struct push { Vreg64 s; };
 636 struct pushm { Vptr s; };
 637 struct ret { RegSet args; };
 638 struct roundsd { RoundDirection dir; VregDbl s, d; };
 639 struct sarq { Vreg64 s, d; VregSF sf; }; // uses rcx
 640 struct sarqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 641 struct setcc { ConditionCode cc; VregSF sf; Vreg8 d; };
 642 struct shlli { Immed s0; Vreg32 s1, d; VregSF sf; };
 643 struct shlq { Vreg64 s, d; VregSF sf; }; // uses rcx
 644 struct shlqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 645 struct shrli { Immed s0; Vreg32 s1, d; VregSF sf; };
 646 struct shrqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 647 struct sqrtsd { VregDbl s, d; };
 648 struct storeb { Vreg8 s; Vptr m; };
 649 struct storebi { Immed s; Vptr m; };
 650 struct storedqu { Vreg128 s; Vptr m; };
 651 struct storel { Vreg32 s; Vptr m; };
 652 struct storeli { Immed s; Vptr m; };
 653 struct storeqi { Immed s; Vptr m; };
 654 struct storesd { VregDbl s; Vptr m; };
 655 struct storew { Vreg16 s; Vptr m; };
 656 struct storewi { Immed s; Vptr m; };
 657 struct subbi { Immed s0; Vreg8 s1, d; VregSF sf; };
 658 struct subl { Vreg32 s0, s1, d; VregSF sf; };
 659 struct subli { Immed s0; Vreg32 s1, d; VregSF sf; };
 660 struct subq { Vreg64 s0, s1, d; VregSF sf; };
 661 struct subqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 662 struct subsd { VregDbl s0, s1, d; };
 663 struct testb { Vreg8 s0, s1; VregSF sf; };
 664 struct testbi { Immed s0; Vreg8 s1; VregSF sf; };
 665 struct testbim { Immed s0; Vptr s1; VregSF sf; };
 666 struct testl { Vreg32 s0, s1; VregSF sf; };
 667 struct testli { Immed s0; Vreg32 s1; VregSF sf; };
 668 struct testlim { Immed s0; Vptr s1; VregSF sf; };
 669 struct testq { Vreg64 s0, s1; VregSF sf; };
 670 struct testqm { Vreg64 s0; Vptr s1; VregSF sf; };
 671 struct testqim { Immed s0; Vptr s1; VregSF sf; };
 672 struct ucomisd { VregDbl s0, s1; VregSF sf; };
 673 struct ud2 {};
 674 struct unpcklpd { VregDbl s0, s1; Vreg128 d; };
 675 struct xorb { Vreg8 s0, s1, d; VregSF sf; };
 676 struct xorbi { Immed s0; Vreg8 s1, d; VregSF sf; };
 677 struct xorq { Vreg64 s0, s1, d; VregSF sf; };
 678 struct xorqi { Immed s0; Vreg64 s1, d; VregSF sf; };
 679
 680 struct Vinstr {
 681 #define O(name, imms, uses, defs) name,
 682   enum Opcode : uint8_t { VASM_OPCODES };
 683 #undef O
 684
 685   Vinstr()
 686     : op(ud2)
 687   {}
 688
 689 #define O(name, imms, uses, defs)                               \
 690   /* implicit */ Vinstr(jit::name i) : op(name), name##_(i) {}
 691   VASM_OPCODES
 692 #undef O
 693
 694   /*
 695    * Define an operator= for all instructions to preserve origin and pos.
 696    */
 697 #define O(name, ...)                            \
 698   Vinstr& operator=(const jit::name& i) {       \
 699     op = Vinstr::name;                          \
 700     name##_ = i;                                \
 701     return *this;                               \
 702   }
 703   VASM_OPCODES
 704 #undef O
 705
 706   template<typename Op>
 707   struct matcher;
 708
 709   /*
 710    * Templated accessors for the union members.
 711    */
 712   template<typename Op>
 713   typename matcher<Op>::type& get() {
 714     return matcher<Op>::get(*this);
 715   }
 716   template<typename Op>
 717   const typename matcher<Op>::type& get() const {
 718     return matcher<Op>::get(*this);
 719   }
 720
 721   Opcode op;
 722
 723   /*
 724    * Instruction position, currently used only in vasm-xls.
 725    */
 726   unsigned pos;
 727
 728   /*
 729    * If present, the IRInstruction this Vinstr was originally created from.
 730    */
 731   const IRInstruction* origin{nullptr};
 732
 733   /*
 734    * A union of all possible instructions, descriminated by the op field.
 735    */
 736 #define O(name, imms, uses, defs) jit::name name##_;
 737   union { VASM_OPCODES };
 738 #undef O
 739 };
 740
 741 #define O(name, ...)                             \
 742   template<> struct Vinstr::matcher<name> {      \
 743     using type = jit::name;                      \
 744     static type& get(Vinstr& inst) {             \
 745       assert(inst.op == name);                   \
 746       return inst.name##_;                       \
 747     }                                            \
 748     static const type& get(const Vinstr& inst) { \
 749       assert(inst.op == name);                   \
 750       return inst.name##_;                       \
 751     }                                            \
 752   };
 753 VASM_OPCODES
 754 #undef O
 755
 756 struct Vblock {
 757   explicit Vblock(AreaIndex area) : area(area) {}
 758   AreaIndex area;
 759   jit::vector<Vinstr> code;
 760 };
 761
 762 typedef jit::vector<Vreg> VregList;
 763
 764 /*
 765  * Source operands for vcall/vinvoke instructions, packed into a struct for
 766  * convenience and to keep the instructions compact.
 767  */
 768 struct VcallArgs {
 769   VregList args, simdArgs, stkArgs;
 770 };
 771
 772 /*
 773  * A Vunit contains all the assets that make up a vasm compilation unit. It is
 774  * responsible for allocating new blocks, Vregs, and tuples.
 775  */
 776 struct Vunit {
 777   /*
 778    * Create a new block in the given area, returning its id.
 779    */
 780   Vlabel makeBlock(AreaIndex area);
 781
 782   /*
 783    * Create a block intended to be used temporarily, as part of modifying
 784    * existing code. Although not necessary for correctness, the block may be
 785    * freed with freeScratchBlock when finished.
 786    */
 787   Vlabel makeScratchBlock();
 788
 789   /*
 790    * Free a scratch block when finished with it. There must be no references to
 791    * this block in reachable code.
 792    */
 793   void freeScratchBlock(Vlabel);
 794
 795   Vreg makeReg() { return Vreg{next_vr++}; }
 796   Vtuple makeTuple(VregList&& regs);
 797   Vtuple makeTuple(const VregList& regs);
 798   VcallArgsId makeVcallArgs(VcallArgs&& args);
 799
 800   Vreg makeConst(bool);
 801   Vreg makeConst(uint64_t);
 802   Vreg makeConst(double);
 803   Vreg makeConst(const void* p) { return makeConst(uint64_t(p)); }
 804   Vreg makeConst(uint32_t v) { return makeConst(uint64_t(v)); }
 805   Vreg makeConst(int64_t v) { return makeConst(uint64_t(v)); }
 806   Vreg makeConst(int32_t v) { return makeConst(int64_t(v)); }
 807   Vreg makeConst(DataType t) { return makeConst(uint64_t(t)); }
 808   Vreg makeConst(Immed64 v) { return makeConst(uint64_t(v.q())); }
 809
 810   template<class T>
 811   typename std::enable_if<std::is_integral<T>::value, Vreg>::type
 812   makeConst(T l) { return makeConst(uint64_t(l)); }
 813
 814   /*
 815    * Returns true iff this Vunit needs register allocation before it can be
 816    * emitted, either because it uses virtual registers or contains instructions
 817    * that must be lowered by xls.
 818    */
 819   bool needsRegAlloc() const;
 820
 821   unsigned next_vr{Vreg::V0};
 822   unsigned next_point{0};
 823   Vlabel entry;
 824   jit::vector<Vblock> blocks;
 825
 826   /*
 827    * Vasm constant: 1 or 8 byte unsigned value.
 828    */
 829   struct Cns {
 830     struct Hash {
 831       size_t operator()(Cns c) const {
 832         return std::hash<uint64_t>()(c.val) ^ c.isByte;
 833       }
 834     };
 835
 836     Cns()
 837       : val(0), isByte(false)
 838     {}
 839
 840     /* implicit */ Cns(bool b)
 841       : val(b), isByte(true) {}
 842
 843     /* implicit */ Cns(uint8_t b)
 844       : val(b), isByte(true) {}
 845
 846     /* implicit */ Cns(uint64_t i)
 847       : val(i), isByte(false) {}
 848
 849     bool operator==(Cns other) const {
 850       return val == other.val && isByte == other.isByte;
 851     }
 852
 853     uint64_t val;
 854     bool isByte;
 855   };
 856
 857   jit::hash_map<Cns,Vreg,Cns::Hash> cpool;
 858   jit::vector<VregList> tuples;
 859   jit::vector<VcallArgs> vcallArgs;
 860 };
 861
 862 // writer stream to add instructions to a block
 863 struct Vout {
 864   Vout(Vunit& u, Vlabel b, const IRInstruction* origin = nullptr)
 865     : m_unit(u), m_block(b), m_origin(origin)
 866   {}
 867
 868   Vout& operator=(const Vout& v) {
 869     assert(&v.m_unit == &m_unit);
 870     m_block = v.m_block;
 871     m_origin = v.m_origin;
 872     return *this;
 873   }
 874
 875   // implicit cast to label for initializing branch instructions
 876   /* implicit */ operator Vlabel() const;
 877   bool empty() const;
 878   bool closed() const;
 879
 880   Vout makeBlock(); // create a stream connected to a new empty block
 881
 882   // instruction emitter
 883   Vout& operator<<(const Vinstr& inst);
 884
 885   Vpoint makePoint() { return Vpoint{m_unit.next_point++}; }
 886   Vunit& unit() { return m_unit; }
 887   template<class T> Vreg cns(T v) { return m_unit.makeConst(v); }
 888   void use(Vlabel b) { m_block = b; }
 889   void setOrigin(const IRInstruction* i) { m_origin = i; }
 890   Vreg makeReg() { return m_unit.makeReg(); }
 891   AreaIndex area() const { return m_unit.blocks[m_block].area; }
 892   Vtuple makeTuple(const VregList& regs) const {
 893     return m_unit.makeTuple(regs);
 894   }
 895   Vtuple makeTuple(VregList&& regs) const {
 896     return m_unit.makeTuple(std::move(regs));
 897   }
 898   VcallArgsId makeVcallArgs(VcallArgs&& args) const {
 899     return m_unit.makeVcallArgs(std::move(args));
 900   }
 901
 902 private:
 903   Vunit& m_unit;
 904   Vlabel m_block;
 905   const IRInstruction* m_origin;
 906 };
 907
 908 // Similar to X64Assembler, but buffers instructions as they
 909 // are written, then generates code all at once at the end.
 910 // Areas represent the separate sections we generate code into;
 911 struct Vasm {
 912   struct Area {
 913     Vout out;
 914     CodeBlock& code;
 915     CodeAddress start;
 916   };
 917   using AreaList = jit::vector<Area>;
 918
 919   explicit Vasm() {
 920     m_areas.reserve(size_t(AreaIndex::Max));
 921   }
 922
 923   void finishX64(const Abi&, AsmInfo* asmInfo);
 924   void finishARM(const Abi&, AsmInfo* asmInfo);
 925
 926   // get an existing area
 927   Vout& main() { return area(AreaIndex::Main).out; }
 928   Vout& cold() { return area(AreaIndex::Cold).out; }
 929   Vout& frozen() { return area(AreaIndex::Frozen).out; }
 930
 931   // create areas
 932   Vout& main(CodeBlock& cb) { return add(cb, AreaIndex::Main); }
 933   Vout& cold(CodeBlock& cb) { return add(cb, AreaIndex::Cold); }
 934   Vout& frozen(CodeBlock& cb) { return add(cb, AreaIndex::Frozen); }
 935   Vout& main(X64Assembler& a) { return main(a.code()); }
 936   Vout& cold(X64Assembler& a) { return cold(a.code()); }
 937   Vout& frozen(X64Assembler& a) { return frozen(a.code()); }
 938   Vunit& unit() { return m_unit; }
 939   AreaList& areas() { return m_areas; }
 940
 941 private:
 942   Vout& add(CodeBlock &cb, AreaIndex area);
 943   Area& area(AreaIndex i) {
 944     assert((unsigned)i < m_areas.size());
 945     return m_areas[(unsigned)i];
 946   }
 947
 948 private:
 949   Vunit m_unit;
 950   AreaList m_areas; // indexed by AreaIndex
 951 };
 952
 953 /*
 954  * Vauto is a convenience helper for emitting small amounts of machine code
 955  * using vasm. It always has a main code block; cold and frozen blocks may be
 956  * added using the normal Vasm API after creation. When the Vauto goes out of
 957  * scope, it will finalize and emit any code it contains.
 958  */
 959 struct Vauto : Vasm {
 960   explicit Vauto(CodeBlock& code) {
 961     unit().entry = Vlabel(main(code));
 962   }
 963   ~Vauto();
 964 };
 965
 966 template<class F> void visit(const Vunit&, Vreg v, F f) {
 967   f(v);
 968 }
 969 template<class F> void visit(const Vunit&, Vptr p, F f) {
 970   if (p.base.isValid()) f(p.base);
 971   if (p.index.isValid()) f(p.index);
 972 }
 973 template<class F> void visit(const Vunit& unit, Vtuple t, F f) {
 974   for (auto r : unit.tuples[t]) f(r);
 975 }
 976 template<class F> void visit(const Vunit& unit, VcallArgsId a, F f) {
 977   auto& args = unit.vcallArgs[a];
 978   for (auto r : args.args) f(r);
 979   for (auto r : args.simdArgs) f(r);
 980   for (auto r : args.stkArgs) f(r);
 981 }
 982 template<class F> void visit(const Vunit& unit, RegSet regs, F f) {
 983   regs.forEach([&](Vreg r) { f(r); });
 984 }
 985
 986 template<class Use>
 987 void visitUses(const Vunit& unit, Vinstr& inst, Use use) {
 988   switch (inst.op) {
 989 #define O(name, imms, uses, defs) \
 990     case Vinstr::name: { \
 991       auto& i = inst.name##_; (void)i; \
 992       uses \
 993       break; \
 994     }
 995 #define U(s) visit(unit, i.s, use);
 996 #define UA(s) visit(unit, i.s, use);
 997 #define UH(s,h) visit(unit, i.s, use);
 998 #define Un
 999     VASM_OPCODES
1000 #undef Un
1001 #undef UH
1002 #undef UA
1003 #undef U
1004 #undef O
1005   }
1006 }
1007
1008 template<class Def>
1009 void visitDefs(const Vunit& unit, const Vinstr& inst, Def def) {
1010   switch (inst.op) {
1011 #define O(name, imms, uses, defs) \
1012     case Vinstr::name: { \
1013       auto& i = inst.name##_; (void)i; \
1014       defs \
1015       break; \
1016     }
1017 #define D(d) visit(unit, i.d, def);
1018 #define DH(d,h) visit(unit, i.d, def);
1019 #define Dn
1020     VASM_OPCODES
1021 #undef Dn
1022 #undef DH
1023 #undef D
1024 #undef O
1025   }
1026 }
1027
1028 /*
1029  * visitOperands visits all operands of the given instruction, calling
1030  * visitor.imm(), visitor.use(), visitor.across(), and visitor.def() as defined
1031  * in the VASM_OPCODES macro.
1032  *
1033  * The template spew is necessary to support callers that only have a const
1034  * Vinstr& as well as callers with a Vinstr& that wish to mutate the
1035  * instruction in the visitor.
1036  */
1037 template<class MaybeConstVinstr, class Visitor>
1038 typename std::enable_if<
1039   std::is_same<MaybeConstVinstr, Vinstr>::value ||
1040   std::is_same<MaybeConstVinstr, const Vinstr>::value
1041 >::type
1042 visitOperands(MaybeConstVinstr& inst, Visitor& visitor) {
1043   switch (inst.op) {
1044 #define O(name, imms, uses, defs) \
1045     case Vinstr::name: { \
1046       auto& i = inst.name##_; (void)i; \
1047       imms \
1048       uses \
1049       defs \
1050       break; \
1051     }
1052 #define I(f) visitor.imm(i.f);
1053 #define U(s) visitor.use(i.s);
1054 #define UA(s) visitor.across(i.s);
1055 #define UH(s,h) visitor.useHint(i.s, i.h);
1056 #define D(d) visitor.def(i.d);
1057 #define DH(d,h) visitor.defHint(i.d, i.h);
1058 #define Inone
1059 #define Un
1060 #define Dn
1061     VASM_OPCODES
1062 #undef Dn
1063 #undef Un
1064 #undef Inone
1065 #undef DH
1066 #undef D
1067 #undef UH
1068 #undef UA
1069 #undef U
1070 #undef I
1071 #undef O
1072   }
1073 }
1074
1075 // visit reachable blocks in postorder, calling fn on each one.
1076 struct PostorderWalker {
1077   template<class Fn> void dfs(Vlabel b, Fn fn) {
1078     if (visited.test(b)) return;
1079     visited.set(b);
1080     for (auto s : succs(unit.blocks[b])) {
1081       dfs(s, fn);
1082     }
1083     fn(b);
1084   }
1085   template<class Fn> void dfs(Fn fn) {
1086     dfs(unit.entry, fn);
1087   }
1088   explicit PostorderWalker(const Vunit& u)
1089     : unit(u)
1090     , visited(u.blocks.size())
1091   {}
1092   const Vunit& unit;
1093   boost::dynamic_bitset<> visited;
1094 };
1095
1096 extern const char* vinst_names[];
1097 bool isBlockEnd(Vinstr& inst);
1098 std::string format(Vreg);
1099 bool check(Vunit&);
1100 bool checkBlockEnd(Vunit& v, Vlabel b);
1101
1102 // search for the phidef in block b, then return its dest tuple
1103 Vtuple findDefs(const Vunit& unit, Vlabel b);
1104
1105 typedef jit::vector<jit::vector<Vlabel>> PredVector;
1106 PredVector computePreds(const Vunit& unit);
1107
1108 }}
1109 #endif