hphp/runtime/vm/jit/code-gen-helpers-x64.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/code-gen-helpers-x64.h"
  18
  19 #include "hphp/util/asm-x64.h"
  20 #include "hphp/util/trace.h"
  21
  22 #include "hphp/runtime/base/runtime-option.h"
  23 #include "hphp/runtime/base/stats.h"
  24 #include "hphp/runtime/base/types.h"
  25 #include "hphp/runtime/vm/jit/translator-inline.h"
  26 #include "hphp/runtime/vm/jit/translator-x64.h"
  27 #include "hphp/runtime/vm/jit/translator-x64-internal.h"
  28 #include "hphp/runtime/vm/jit/translator.h"
  29 #include "hphp/runtime/vm/jit/x64-util.h"
  30 #include "hphp/runtime/vm/jit/ir.h"
  31
  32 namespace HPHP {
  33 namespace JIT {
  34
  35 namespace {
  36
  37 //////////////////////////////////////////////////////////////////////
  38
  39 using namespace Util;
  40 using namespace Transl;
  41 using namespace Transl::reg;
  42
  43 TRACE_SET_MOD(hhir);
  44
  45 } // unnamed namespace
  46
  47 namespace CodeGenHelpersX64 {
  48
  49 /*
  50  * It's not normally ok to directly use tracelet abi registers in
  51  * codegen, unless you're directly dealing with an instruction that
  52  * does near-end-of-tracelet glue.  (Or also we sometimes use them
  53  * just for some static_assertions relating to calls to helpers from
  54  * tx64 that hardcode these registers.)
  55  */
  56 using Transl::rVmFp;
  57 using Transl::rVmSp;
  58
  59 /*
  60  * Satisfy an alignment constraint. If we're in a reachable section
  61  * of code, bridge the gap with nops. Otherwise, int3's.
  62  */
  63 void moveToAlign(Asm& aa,
  64                  const size_t align /* =kJmpTargetAlign */,
  65                  const bool unreachable /* =true */) {
  66   using namespace HPHP::Util;
  67   assert(isPowerOfTwo(align));
  68   size_t leftInBlock = align - ((align - 1) & uintptr_t(aa.frontier()));
  69   if (leftInBlock == align) return;
  70   if (unreachable) {
  71     if (leftInBlock > 2) {
  72       aa.ud2();
  73       leftInBlock -= 2;
  74     }
  75     if (leftInBlock > 0) {
  76       aa.emitInt3s(leftInBlock);
  77     }
  78     return;
  79   }
  80   aa.emitNop(leftInBlock);
  81 }
  82
  83 void emitEagerSyncPoint(Asm& as, const HPHP::Opcode* pc, const Offset spDiff) {
  84   static COff spOff = offsetof(VMExecutionContext, m_stack) +
  85     Stack::topOfStackOffset();
  86   static COff fpOff = offsetof(VMExecutionContext, m_fp);
  87   static COff pcOff = offsetof(VMExecutionContext, m_pc);
  88
  89   /* we can't use rAsm because the pc store uses it as a
  90      temporary */
  91   Reg64 rEC = reg::rdi;
  92
  93   as.  push(rEC);
  94   emitGetGContext(as, rEC);
  95   as.  storeq(rVmFp, rEC[fpOff]);
  96   if (spDiff) {
  97     as.  lea(rVmSp[spDiff], rAsm);
  98     as.  storeq(rAsm, rEC[spOff]);
  99   } else {
 100     as.  storeq(rVmSp, rEC[spOff]);
 101   }
 102   as.  storeq(pc, rEC[pcOff]);
 103   as.  pop(rEC);
 104 }
 105
 106 // emitEagerVMRegSave --
 107 //   Inline. Saves regs in-place in the TC. This is an unusual need;
 108 //   you probably want to lazily save these regs via recordCall and
 109 //   its ilk.
 110 void emitEagerVMRegSave(Asm& as, RegSaveFlags flags) {
 111   bool saveFP = bool(flags & RegSaveFlags::SaveFP);
 112   bool savePC = bool(flags & RegSaveFlags::SavePC);
 113   assert((flags & ~(RegSaveFlags::SavePC | RegSaveFlags::SaveFP)) ==
 114          RegSaveFlags::None);
 115
 116   Reg64 pcReg = rdi;
 117   PhysReg rEC = rAsm;
 118   assert(!kSpecialCrossTraceRegs.contains(rdi));
 119
 120   emitGetGContext(as, rEC);
 121
 122   static COff spOff = offsetof(VMExecutionContext, m_stack) +
 123     Stack::topOfStackOffset();
 124   static COff fpOff = offsetof(VMExecutionContext, m_fp) - spOff;
 125   static COff pcOff = offsetof(VMExecutionContext, m_pc) - spOff;
 126
 127   assert(spOff != 0);
 128   as.   addq   (spOff, r64(rEC));
 129   as.   storeq (rVmSp, *rEC);
 130   if (savePC) {
 131     // We're going to temporarily abuse rVmSp to hold the current unit.
 132     Reg64 rBC = rVmSp;
 133     as. push   (rBC);
 134     // m_fp -> m_func -> m_unit -> m_bc + pcReg
 135     as. loadq  (rVmFp[AROFF(m_func)], rBC);
 136     as. loadq  (rBC[Func::unitOff()], rBC);
 137     as. loadq  (rBC[Unit::bcOff()], rBC);
 138     as. addq   (rBC, pcReg);
 139     as. storeq (pcReg, rEC[pcOff]);
 140     as. pop    (rBC);
 141   }
 142   if (saveFP) {
 143     as. storeq (rVmFp, rEC[fpOff]);
 144   }
 145 }
 146
 147 void emitGetGContext(Asm& as, PhysReg dest) {
 148   emitTLSLoad<ExecutionContext>(as, g_context, dest);
 149 }
 150
 151 // IfCountNotStatic --
 152 //   Emits if (%reg->_count != RefCountStaticValue) { ... }.
 153 //   May short-circuit this check if the type is known to be
 154 //   static already.
 155 struct IfCountNotStatic {
 156   typedef CondBlock<FAST_REFCOUNT_OFFSET,
 157                     RefCountStaticValue,
 158                     CC_Z,
 159                     field_type(RefData, m_count)> NonStaticCondBlock;
 160   NonStaticCondBlock *m_cb; // might be null
 161   IfCountNotStatic(Asm& as,
 162                    PhysReg reg,
 163                    DataType t = KindOfInvalid) {
 164
 165     // Objects and variants cannot be static
 166     if (t != KindOfObject && t != KindOfResource && t != KindOfRef) {
 167       m_cb = new NonStaticCondBlock(as, reg);
 168     } else {
 169       m_cb = nullptr;
 170     }
 171   }
 172
 173   ~IfCountNotStatic() {
 174     delete m_cb;
 175   }
 176 };
 177
 178 void emitIncRef(Asm& as, PhysReg base) {
 179   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 180     emitAssertRefCount(as, base);
 181   }
 182   // emit incref
 183   as.incl(base[FAST_REFCOUNT_OFFSET]);
 184   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 185     // Assert that the ref count is greater than zero
 186     emitAssertFlagsNonNegative(as);
 187   }
 188 }
 189
 190 void emitIncRefCheckNonStatic(Asm& as, PhysReg base, DataType dtype) {
 191   { // if !static then
 192     IfCountNotStatic ins(as, base, dtype);
 193     emitIncRef(as, base);
 194   } // endif
 195 }
 196
 197 void emitIncRefGenericRegSafe(Asm& as, PhysReg base, int disp, PhysReg tmpReg) {
 198   { // if RC
 199     IfRefCounted irc(as, base, disp);
 200     as.   load_reg64_disp_reg64(base, disp + TVOFF(m_data),
 201                                 tmpReg);
 202     { // if !static
 203       IfCountNotStatic ins(as, tmpReg);
 204       as. incl(tmpReg[FAST_REFCOUNT_OFFSET]);
 205     } // endif
 206   } // endif
 207 }
 208
 209 void emitAssertFlagsNonNegative(Asm& as) {
 210   ifThen(as, CC_NGE, [&] { as.ud2(); });
 211 }
 212
 213 void emitAssertRefCount(Asm& as, PhysReg base) {
 214   as.cmpl(HPHP::RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
 215   ifThen(as, CC_NBE, [&] { as.ud2(); });
 216 }
 217
 218 // Logical register move: ensures the value in src will be in dest
 219 // after execution, but might do so in strange ways. Do not count on
 220 // being able to smash dest to a different register in the future, e.g.
 221 void emitMovRegReg(Asm& as, PhysReg srcReg, PhysReg dstReg) {
 222   assert(srcReg != InvalidReg);
 223   assert(dstReg != InvalidReg);
 224
 225   if (srcReg == dstReg) return;
 226
 227   if (srcReg.isGP()) {
 228     if (dstReg.isGP()) {                 // GP => GP
 229       as. movq(srcReg, dstReg);
 230     } else {                             // GP => XMM
 231       // This generates a movq x86 instruction, which zero extends
 232       // the 64-bit value in srcReg into a 128-bit XMM register
 233       as. mov_reg64_xmm(srcReg, dstReg);
 234     }
 235   } else {
 236     if (dstReg.isGP()) {                 // XMM => GP
 237       as. mov_xmm_reg64(srcReg, dstReg);
 238     } else {                             // XMM => XMM
 239       // This copies all 128 bits in XMM,
 240       // thus avoiding partial register stalls
 241       as. movdqa(srcReg, dstReg);
 242     }
 243   }
 244 }
 245
 246 void emitLea(Asm& as, PhysReg base, int disp, PhysReg dest) {
 247   if (disp == 0) {
 248     emitMovRegReg(as, base, dest);
 249   } else {
 250     as. lea(base[disp], dest);
 251   }
 252 }
 253
 254 void emitLea(Asm& as, MemoryRef mr, PhysReg dst) {
 255   if (dst == InvalidReg) return;
 256   if (mr.r.disp == 0) {
 257     emitMovRegReg(as, mr.r.base, dst);
 258   } else {
 259     as. lea(mr, dst);
 260   }
 261 }
 262
 263 void emitLdObjClass(Asm& as, PhysReg objReg, PhysReg dstReg) {
 264   as.   loadq (objReg[ObjectData::getVMClassOffset()], dstReg);
 265 }
 266
 267 void emitLdClsCctx(Asm& as, PhysReg srcReg, PhysReg dstReg) {
 268   emitMovRegReg(as, srcReg, dstReg);
 269   as.   decq(dstReg);
 270 }
 271
 272 void emitExitSlowStats(Asm& as, const Func* func, SrcKey dest) {
 273   if (RuntimeOption::EnableInstructionCounts ||
 274       HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) {
 275     Stats::emitInc(as,
 276                    Stats::opcodeToIRPreStatCounter(
 277                      Op(*func->unit()->at(dest.offset()))),
 278                    -1,
 279                    Transl::CC_None,
 280                    true);
 281   }
 282 }
 283
 284 void shuffle2(Asm& as, PhysReg s0, PhysReg s1, PhysReg d0, PhysReg d1) {
 285   assert(s0 != s1);
 286   if (d0 == s1 && d1 != InvalidReg) {
 287     assert(d0 != d1);
 288     if (d1 == s0) {
 289       as.   xchgq (s1, s0);
 290     } else {
 291       as.   movq (s1, d1); // save s1 first; d1 != s0
 292       as.   movq (s0, d0);
 293     }
 294   } else {
 295     if (d0 != InvalidReg) emitMovRegReg(as, s0, d0); // d0 != s1
 296     if (d1 != InvalidReg) emitMovRegReg(as, s1, d1);
 297   }
 298 }
 299
 300 void zeroExtendIfBool(CodeGenerator::Asm& as, const SSATmp* src, PhysReg reg) {
 301   if (src->isA(Type::Bool) && reg != InvalidReg) {
 302     // zero-extend the bool from a byte to a quad
 303     // note: movzbl actually extends the value to 64 bits.
 304     as.movzbl(rbyte(reg), r32(reg));
 305   }
 306 }
 307
 308 ConditionCode opToConditionCode(Opcode opc) {
 309   using namespace HPHP::Transl;
 310
 311   switch (opc) {
 312   case JmpGt:                 return CC_G;
 313   case JmpGte:                return CC_GE;
 314   case JmpLt:                 return CC_L;
 315   case JmpLte:                return CC_LE;
 316   case JmpEq:                 return CC_E;
 317   case JmpNeq:                return CC_NE;
 318   case JmpSame:               return CC_E;
 319   case JmpNSame:              return CC_NE;
 320   case JmpInstanceOfBitmask:  return CC_NZ;
 321   case JmpNInstanceOfBitmask: return CC_Z;
 322   case JmpIsType:             return CC_NZ;
 323   case JmpIsNType:            return CC_Z;
 324   case JmpZero:               return CC_Z;
 325   case JmpNZero:              return CC_NZ;
 326   case ReqBindJmpGt:                 return CC_G;
 327   case ReqBindJmpGte:                return CC_GE;
 328   case ReqBindJmpLt:                 return CC_L;
 329   case ReqBindJmpLte:                return CC_LE;
 330   case ReqBindJmpEq:                 return CC_E;
 331   case ReqBindJmpNeq:                return CC_NE;
 332   case ReqBindJmpSame:               return CC_E;
 333   case ReqBindJmpNSame:              return CC_NE;
 334   case ReqBindJmpInstanceOfBitmask:  return CC_NZ;
 335   case ReqBindJmpNInstanceOfBitmask: return CC_Z;
 336   case ReqBindJmpZero:               return CC_Z;
 337   case ReqBindJmpNZero:              return CC_NZ;
 338   default:
 339     always_assert(0);
 340   }
 341 }
 342
 343 /*
 344  * emitServiceReqWork --
 345  *
 346  *   Call a translator service co-routine. The code emitted here
 347  *   reenters the enterTC loop, invoking the requested service. Control
 348  *   will be returned non-locally to the next logical instruction in
 349  *   the TC.
 350  *
 351  *   Return value is a destination; we emit the bulky service
 352  *   request code into astubs.
 353  *
 354  *   Returns a continuation that will run after the arguments have been
 355  *   emitted. This is gross, but is a partial workaround for the inability
 356  *   to capture argument packs in the version of gcc we're using.
 357  */
 358 TCA
 359 emitServiceReqWork(Asm& as, TCA start, bool persist, SRFlags flags,
 360                    ServiceRequest req, const ServiceReqArgVec& argv) {
 361   assert(start);
 362   const bool align = flags & SRFlags::Align;
 363
 364   /*
 365    * Remember previous state of the code cache.
 366    */
 367   boost::optional<CodeCursor> maybeCc = boost::none;
 368   if (start != as.frontier()) {
 369     maybeCc = boost::in_place<CodeCursor>(boost::ref(as), start);
 370   }
 371
 372   /* max space for moving to align, saving VM regs plus emitting args */
 373   static const int
 374     kVMRegSpace = 0x14,
 375     kMovSize = 0xa,
 376     kNumServiceRegs = sizeof(serviceReqArgRegs) / sizeof(PhysReg),
 377     kMaxStubSpace = kJmpTargetAlign - 1 + kVMRegSpace +
 378       kNumServiceRegs * kMovSize;
 379   if (align) {
 380     moveToAlign(as);
 381   }
 382   TCA retval = as.frontier();
 383   TRACE(3, "Emit Service Req @%p %s(", start, serviceReqName(req));
 384   /*
 385    * Move args into appropriate regs. Eager VMReg save may bash flags,
 386    * so set the CondCode arguments first.
 387    */
 388   for (int i = 0; i < argv.size(); ++i) {
 389     assert(i < kNumServiceReqArgRegs);
 390     auto reg = serviceReqArgRegs[i];
 391     const auto& argInfo = argv[i];
 392     switch(argv[i].m_kind) {
 393       case ServiceReqArgInfo::Immediate: {
 394         TRACE(3, "%" PRIx64 ", ", argInfo.m_imm);
 395         as.    emitImmReg(argInfo.m_imm, reg);
 396       } break;
 397       case ServiceReqArgInfo::CondCode: {
 398         // Already set before VM reg save.
 399         DEBUG_ONLY TCA start = as.frontier();
 400         as.    setcc(argInfo.m_cc, rbyte(reg));
 401         assert(start - as.frontier() <= kMovSize);
 402         TRACE(3, "cc(%x), ", argInfo.m_cc);
 403       } break;
 404       default: not_reached();
 405     }
 406   }
 407   emitEagerVMRegSave(as, JIT::RegSaveFlags::SaveFP);
 408   if (persist) {
 409     as.  emitImmReg(0, rAsm);
 410   } else {
 411     as.  emitImmReg((uint64_t)start, rAsm);
 412   }
 413   TRACE(3, ")\n");
 414   as.    emitImmReg(req, rdi);
 415
 416   /*
 417    * Weird hand-shaking with enterTC: reverse-call a service routine.
 418    *
 419    * In the case of some special stubs (m_callToExit, m_retHelper), we
 420    * have already unbalanced the return stack by doing a ret to
 421    * something other than enterTCHelper.  In that case
 422    * SRJmpInsteadOfRet indicates to fake the return.
 423    */
 424   if (flags & SRFlags::JmpInsteadOfRet) {
 425     as.  pop(rax);
 426     as.  jmp(rax);
 427   } else {
 428     as.  ret();
 429   }
 430
 431   // TODO(2796856): we should record an OpServiceRequest pseudo-bytecode here.
 432
 433   translator_not_reached(as);
 434   if (!persist) {
 435     /*
 436      * Recycled stubs need to be uniformly sized. Make space for the
 437      * maximal possible service requests.
 438      */
 439     assert(as.frontier() - start <= kMaxStubSpace);
 440     as.emitNop(start + kMaxStubSpace - as.frontier());
 441     assert(as.frontier() - start == kMaxStubSpace);
 442   }
 443   return retval;
 444 }
 445
 446 }}}