hphp/runtime/vm/jit/code-gen-helpers-x64.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/code-gen-helpers-x64.h"
  18
  19 #include "hphp/util/asm-x64.h"
  20 #include "hphp/util/ringbuffer.h"
  21 #include "hphp/util/trace.h"
  22
  23 #include "hphp/runtime/base/runtime-option.h"
  24 #include "hphp/runtime/base/stats.h"
  25 #include "hphp/runtime/base/types.h"
  26 #include "hphp/runtime/vm/jit/arch.h"
  27 #include "hphp/runtime/vm/jit/jump-smash.h"
  28 #include "hphp/runtime/vm/jit/translator-inline.h"
  29 #include "hphp/runtime/vm/jit/mc-generator.h"
  30 #include "hphp/runtime/vm/jit/mc-generator-internal.h"
  31 #include "hphp/runtime/vm/jit/translator.h"
  32 #include "hphp/runtime/vm/jit/ir.h"
  33 #include "hphp/runtime/vm/jit/code-gen-x64.h"
  34
  35 namespace HPHP { namespace JIT { namespace X64 {
  36
  37 //////////////////////////////////////////////////////////////////////
  38
  39 using namespace JIT::reg;
  40
  41 TRACE_SET_MOD(hhir);
  42
  43 //////////////////////////////////////////////////////////////////////
  44
  45 /*
  46  * It's not normally ok to directly use tracelet abi registers in
  47  * codegen, unless you're directly dealing with an instruction that
  48  * does near-end-of-tracelet glue.  (Or also we sometimes use them
  49  * just for some static_assertions relating to calls to helpers from
  50  * mcg that hardcode these registers.)
  51  */
  52
  53 /*
  54  * Satisfy an alignment constraint. Bridge the gap with int3's.
  55  */
  56 void moveToAlign(CodeBlock& cb,
  57                  const size_t align /* =kJmpTargetAlign */) {
  58   switch (arch()) {
  59     case Arch::X64: {
  60       X64Assembler a { cb };
  61       assert(folly::isPowTwo(align));
  62       size_t leftInBlock = align - ((align - 1) & uintptr_t(cb.frontier()));
  63       if (leftInBlock == align) return;
  64       if (leftInBlock > 2) {
  65         a.ud2();
  66         leftInBlock -= 2;
  67       }
  68       break;
  69     }
  70     case Arch::ARM:
  71       // TODO(2967396) implement properly, move function
  72       break;
  73   }
  74 }
  75
  76 void emitEagerSyncPoint(Asm& as, const Op* pc) {
  77   static COff spOff = offsetof(ExecutionContext, m_stack) +
  78     Stack::topOfStackOffset();
  79   static COff fpOff = offsetof(ExecutionContext, m_fp);
  80   static COff pcOff = offsetof(ExecutionContext, m_pc);
  81
  82   // we can use rAsm because we don't clobber it in X64Assembler
  83   Reg64 rEC = rAsm;
  84   emitGetGContext(as, rEC);
  85   as.  storeq(rVmFp, rEC[fpOff]);
  86   as.  storeq(rVmSp, rEC[spOff]);
  87   emitImmStoreq(as, intptr_t(pc), rEC[pcOff]);
  88 }
  89
  90 // emitEagerVMRegSave --
  91 //   Inline. Saves regs in-place in the TC. This is an unusual need;
  92 //   you probably want to lazily save these regs via recordCall and
  93 //   its ilk.
  94 void emitEagerVMRegSave(Asm& as, RegSaveFlags flags) {
  95   bool saveFP = bool(flags & RegSaveFlags::SaveFP);
  96   bool savePC = bool(flags & RegSaveFlags::SavePC);
  97   assert((flags & ~(RegSaveFlags::SavePC | RegSaveFlags::SaveFP)) ==
  98          RegSaveFlags::None);
  99
 100   Reg64 pcReg = rdi;
 101   PhysReg rEC = rAsm;
 102   assert(!kSpecialCrossTraceRegs.contains(rdi));
 103
 104   emitGetGContext(as, rEC);
 105
 106   static COff spOff = offsetof(ExecutionContext, m_stack) +
 107     Stack::topOfStackOffset();
 108   static COff fpOff = offsetof(ExecutionContext, m_fp) - spOff;
 109   static COff pcOff = offsetof(ExecutionContext, m_pc) - spOff;
 110
 111   assert(spOff != 0);
 112   as.   addq   (spOff, r64(rEC));
 113   as.   storeq (rVmSp, *rEC);
 114   if (savePC) {
 115     // We're going to temporarily abuse rVmSp to hold the current unit.
 116     Reg64 rBC = rVmSp;
 117     as. push   (rBC);
 118     // m_fp -> m_func -> m_unit -> m_bc + pcReg
 119     as. loadq  (rVmFp[AROFF(m_func)], rBC);
 120     as. loadq  (rBC[Func::unitOff()], rBC);
 121     as. loadq  (rBC[Unit::bcOff()], rBC);
 122     as. addq   (rBC, pcReg);
 123     as. storeq (pcReg, rEC[pcOff]);
 124     as. pop    (rBC);
 125   }
 126   if (saveFP) {
 127     as. storeq (rVmFp, rEC[fpOff]);
 128   }
 129 }
 130
 131 void emitGetGContext(Asm& as, PhysReg dest) {
 132   emitTLSLoad<ExecutionContext>(as, g_context, dest);
 133 }
 134
 135 // IfCountNotStatic --
 136 //   Emits if (%reg->_count < 0) { ... }.
 137 //   This depends on UncountedValue and StaticValue
 138 //   being the only valid negative refCounts and both indicating no
 139 //   ref count is needed.
 140 //   May short-circuit this check if the type is known to be
 141 //   static already.
 142 struct IfCountNotStatic {
 143   typedef CondBlock<FAST_REFCOUNT_OFFSET,
 144                     0,
 145                     CC_S,
 146                     int32_t> NonStaticCondBlock;
 147   static_assert(UncountedValue < 0 && StaticValue < 0, "");
 148   NonStaticCondBlock *m_cb; // might be null
 149   IfCountNotStatic(Asm& as,
 150                    PhysReg reg,
 151                    DataType t = KindOfInvalid) {
 152
 153     // Objects and variants cannot be static
 154     if (t != KindOfObject && t != KindOfResource && t != KindOfRef) {
 155       m_cb = new NonStaticCondBlock(as, reg);
 156     } else {
 157       m_cb = nullptr;
 158     }
 159   }
 160
 161   ~IfCountNotStatic() {
 162     delete m_cb;
 163   }
 164 };
 165
 166
 167 void emitTransCounterInc(Asm& a) {
 168   if (!tx->isTransDBEnabled()) return;
 169
 170   a.    movq (tx->getTransCounterAddr(), rAsm);
 171   a.    lock ();
 172   a.    incq (*rAsm);
 173 }
 174
 175 void emitIncRef(Asm& as, PhysReg base) {
 176   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 177     emitAssertRefCount(as, base);
 178   }
 179   // emit incref
 180   as.incl(base[FAST_REFCOUNT_OFFSET]);
 181   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 182     // Assert that the ref count is greater than zero
 183     emitAssertFlagsNonNegative(as);
 184   }
 185 }
 186
 187 void emitIncRefCheckNonStatic(Asm& as, PhysReg base, DataType dtype) {
 188   { // if !static then
 189     IfCountNotStatic ins(as, base, dtype);
 190     emitIncRef(as, base);
 191   } // endif
 192 }
 193
 194 void emitIncRefGenericRegSafe(Asm& as, PhysReg base, int disp, PhysReg tmpReg) {
 195   { // if RC
 196     IfRefCounted irc(as, base, disp);
 197     as.   loadq  (base[disp + TVOFF(m_data)], tmpReg);
 198     { // if !static
 199       IfCountNotStatic ins(as, tmpReg);
 200       as. incl(tmpReg[FAST_REFCOUNT_OFFSET]);
 201     } // endif
 202   } // endif
 203 }
 204
 205 void emitAssertFlagsNonNegative(Asm& as) {
 206   ifThen(as, CC_NGE, [&] { as.ud2(); });
 207 }
 208
 209 void emitAssertRefCount(Asm& as, PhysReg base) {
 210   as.cmpl(HPHP::StaticValue, base[FAST_REFCOUNT_OFFSET]);
 211   ifThen(as, CC_NBE, [&] {
 212       as.cmpl(HPHP::RefCountMaxRealistic, base[FAST_REFCOUNT_OFFSET]);
 213       ifThen(as, CC_NBE, [&] { as.ud2(); });
 214   });
 215 }
 216
 217 // Logical register move: ensures the value in src will be in dest
 218 // after execution, but might do so in strange ways. Do not count on
 219 // being able to smash dest to a different register in the future, e.g.
 220 void emitMovRegReg(Asm& as, PhysReg srcReg, PhysReg dstReg) {
 221   assert(srcReg != InvalidReg);
 222   assert(dstReg != InvalidReg);
 223
 224   if (srcReg == dstReg) return;
 225
 226   if (srcReg.isGP()) {
 227     if (dstReg.isGP()) {                 // GP => GP
 228       as. movq(srcReg, dstReg);
 229     } else {                             // GP => XMM
 230       // This generates a movq x86 instruction, which zero extends
 231       // the 64-bit value in srcReg into a 128-bit XMM register
 232       as. movq_rx(srcReg, dstReg);
 233     }
 234   } else {
 235     if (dstReg.isGP()) {                 // XMM => GP
 236       as. movq_xr(srcReg, dstReg);
 237     } else {                             // XMM => XMM
 238       // This copies all 128 bits in XMM,
 239       // thus avoiding partial register stalls
 240       as. movdqa(srcReg, dstReg);
 241     }
 242   }
 243 }
 244
 245 void emitLea(Asm& as, MemoryRef mr, PhysReg dst) {
 246   if (dst == InvalidReg) return;
 247   if (mr.r.disp == 0) {
 248     emitMovRegReg(as, mr.r.base, dst);
 249   } else {
 250     as. lea(mr, dst);
 251   }
 252 }
 253
 254 void emitLdObjClass(Asm& as, PhysReg objReg, PhysReg dstReg) {
 255   as.   loadq (objReg[ObjectData::getVMClassOffset()], dstReg);
 256 }
 257
 258 void emitLdClsCctx(Asm& as, PhysReg srcReg, PhysReg dstReg) {
 259   emitMovRegReg(as, srcReg, dstReg);
 260   as.   decq(dstReg);
 261 }
 262
 263 void emitCall(Asm& a, TCA dest) {
 264   if (a.jmpDeltaFits(dest) && !Stats::enabled()) {
 265     a.    call(dest);
 266   } else {
 267     a.    call(mcg->getNativeTrampoline(dest));
 268   }
 269 }
 270
 271 void emitCall(Asm& a, CppCall call) {
 272   if (call.isDirect()) {
 273     return emitCall(a, (TCA)call.getAddress());
 274   } else if (call.isVirtual()) {
 275     // Virtual call.
 276     // Load method's address from proper offset off of object in rdi,
 277     // using rax as scratch.
 278     a.  loadq  (*rdi, rax);
 279     a.  call   (rax[call.getOffset()]);
 280   } else {
 281     assert(call.isIndirect());
 282     a.  call   (call.getReg());
 283   }
 284 }
 285
 286 void emitJmpOrJcc(Asm& a, ConditionCode cc, TCA dest) {
 287   if (cc == CC_None) {
 288     a.   jmp(dest);
 289   } else {
 290     a.   jcc((ConditionCode)cc, dest);
 291   }
 292 }
 293
 294 void emitRB(X64Assembler& a,
 295             Trace::RingBufferType t,
 296             const char* msg,
 297             RegSet toSave) {
 298   if (!Trace::moduleEnabledRelease(Trace::ringbuffer, 1)) {
 299     return;
 300   }
 301   PhysRegSaver save(a, toSave | kSpecialCrossTraceRegs);
 302   int arg = 0;
 303   a.    emitImmReg((uintptr_t)msg, argNumToRegName[arg++]);
 304   a.    emitImmReg(strlen(msg), argNumToRegName[arg++]);
 305   a.    emitImmReg(t, argNumToRegName[arg++]);
 306   a.    call((TCA)Trace::ringbufferMsg);
 307 }
 308
 309 void emitTraceCall(CodeBlock& cb, int64_t pcOff) {
 310   switch (arch()) {
 311     case Arch::X64: {
 312       Asm a { cb };
 313       // call to a trace function
 314       a.    movq   (a.frontier(), rcx);
 315       a.    movq   (rVmFp, rdi);
 316       a.    movq   (rVmSp, rsi);
 317       a.    movq   (pcOff, rdx);
 318       // do the call; may use a trampoline
 319       emitCall(a, reinterpret_cast<TCA>(traceCallback));
 320       break;
 321     }
 322     case Arch::ARM:
 323       // TODO(2967396) implement properly, move function
 324       break;
 325   }
 326 }
 327
 328 void emitTestSurpriseFlags(Asm& a) {
 329   static_assert(RequestInjectionData::LastFlag < (1 << 8),
 330                 "Translator assumes RequestInjectionFlags fit in one byte");
 331   a.    testb((int8_t)0xff, rVmTl[RDS::kConditionFlagsOff]);
 332 }
 333
 334 void emitCheckSurpriseFlagsEnter(CodeBlock& mainCode, CodeBlock& stubsCode,
 335                                  bool inTracelet, FixupMap& fixupMap,
 336                                  Fixup fixup) {
 337   Asm a { mainCode };
 338   Asm astubs { stubsCode };
 339
 340   emitTestSurpriseFlags(a);
 341   a.  jnz  (stubsCode.frontier());
 342
 343   astubs.  movq  (rVmFp, argNumToRegName[0]);
 344   emitCall(astubs, tx->uniqueStubs.functionEnterHelper);
 345   if (inTracelet) {
 346     fixupMap.recordSyncPoint(stubsCode.frontier(),
 347                              fixup.m_pcOffset, fixup.m_spOffset);
 348   } else {
 349     // If we're being called while generating a func prologue, we
 350     // have to record the fixup directly in the fixup map instead of
 351     // going through the pending fixup path like normal.
 352     fixupMap.recordFixup(stubsCode.frontier(), fixup);
 353   }
 354   astubs.  jmp   (mainCode.frontier());
 355 }
 356
 357 void shuffle2(Asm& as, PhysReg s0, PhysReg s1, PhysReg d0, PhysReg d1) {
 358   if (s0 == InvalidReg && s1 == InvalidReg &&
 359       d0 == InvalidReg && d1 == InvalidReg) return;
 360   assert(s0 != s1);
 361   assert(!s0.isSIMD() || s1 == InvalidReg); // never 2 XMMs
 362   assert(!d0.isSIMD() || d1 == InvalidReg); // never 2 XMMs
 363   if (d0 == s1 && d1 != InvalidReg) {
 364     assert(d0 != d1);
 365     if (d1 == s0) {
 366       as.   xchgq (s1, s0);
 367     } else {
 368       as.   movq (s1, d1); // save s1 first; d1 != s0
 369       as.   movq (s0, d0);
 370     }
 371   } else if (d0.isSIMD() && s0.isGP() && s1.isGP()) {
 372     // move 2 gpr to 1 xmm
 373     assert(d0 != rCgXMM0); // xmm0 is reserved for scratch
 374     as.   movq_rx(s0, d0);
 375     as.   movq_rx(s1, rCgXMM0);
 376     as.   unpcklpd(rCgXMM0, d0); // s1 -> d0[1]
 377   } else {
 378     if (d0 != InvalidReg) emitMovRegReg(as, s0, d0); // d0 != s1
 379     if (d1 != InvalidReg) emitMovRegReg(as, s1, d1);
 380   }
 381 }
 382
 383 void zeroExtendIfBool(CodeGenerator::Asm& as, const SSATmp* src, PhysReg reg) {
 384   if (src->isA(Type::Bool) && reg != InvalidReg) {
 385     // zero-extend the bool from a byte to a quad
 386     // note: movzbl actually extends the value to 64 bits.
 387     as.movzbl(rbyte(reg), r32(reg));
 388   }
 389 }
 390
 391 ConditionCode opToConditionCode(Opcode opc) {
 392   switch (opc) {
 393   case JmpGt:                 return CC_G;
 394   case JmpGte:                return CC_GE;
 395   case JmpLt:                 return CC_L;
 396   case JmpLte:                return CC_LE;
 397   case JmpEq:                 return CC_E;
 398   case JmpNeq:                return CC_NE;
 399   case JmpGtInt:              return CC_G;
 400   case JmpGteInt:             return CC_GE;
 401   case JmpLtInt:              return CC_L;
 402   case JmpLteInt:             return CC_LE;
 403   case JmpEqInt:              return CC_E;
 404   case JmpNeqInt:             return CC_NE;
 405   case JmpSame:               return CC_E;
 406   case JmpNSame:              return CC_NE;
 407   case JmpInstanceOfBitmask:  return CC_NZ;
 408   case JmpNInstanceOfBitmask: return CC_Z;
 409   case JmpZero:               return CC_Z;
 410   case JmpNZero:              return CC_NZ;
 411   case ReqBindJmpGt:                 return CC_G;
 412   case ReqBindJmpGte:                return CC_GE;
 413   case ReqBindJmpLt:                 return CC_L;
 414   case ReqBindJmpLte:                return CC_LE;
 415   case ReqBindJmpEq:                 return CC_E;
 416   case ReqBindJmpNeq:                return CC_NE;
 417   case ReqBindJmpGtInt:              return CC_G;
 418   case ReqBindJmpGteInt:             return CC_GE;
 419   case ReqBindJmpLtInt:              return CC_L;
 420   case ReqBindJmpLteInt:             return CC_LE;
 421   case ReqBindJmpEqInt:              return CC_E;
 422   case ReqBindJmpNeqInt:             return CC_NE;
 423   case ReqBindJmpSame:               return CC_E;
 424   case ReqBindJmpNSame:              return CC_NE;
 425   case ReqBindJmpInstanceOfBitmask:  return CC_NZ;
 426   case ReqBindJmpNInstanceOfBitmask: return CC_Z;
 427   case ReqBindJmpZero:               return CC_Z;
 428   case ReqBindJmpNZero:              return CC_NZ;
 429   case SideExitJmpGt:                 return CC_G;
 430   case SideExitJmpGte:                return CC_GE;
 431   case SideExitJmpLt:                 return CC_L;
 432   case SideExitJmpLte:                return CC_LE;
 433   case SideExitJmpEq:                 return CC_E;
 434   case SideExitJmpNeq:                return CC_NE;
 435   case SideExitJmpGtInt:              return CC_G;
 436   case SideExitJmpGteInt:             return CC_GE;
 437   case SideExitJmpLtInt:              return CC_L;
 438   case SideExitJmpLteInt:             return CC_LE;
 439   case SideExitJmpEqInt:              return CC_E;
 440   case SideExitJmpNeqInt:             return CC_NE;
 441   case SideExitJmpSame:               return CC_E;
 442   case SideExitJmpNSame:              return CC_NE;
 443   case SideExitJmpInstanceOfBitmask:  return CC_NZ;
 444   case SideExitJmpNInstanceOfBitmask: return CC_Z;
 445   case SideExitJmpZero:               return CC_Z;
 446   case SideExitJmpNZero:              return CC_NZ;
 447   default:
 448     always_assert(0);
 449   }
 450 }
 451
 452 }}}