src/runtime/vm/translator/translator-x64-internal.h

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010- Facebook, Inc. (http://www.facebook.com)         |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #ifndef incl_TRANSLATOR_X64_INTERNAL_H_
  17 #define incl_TRANSLATOR_X64_INTERNAL_H_
  18
  19 #include <boost/optional.hpp>
  20 #include <boost/filesystem.hpp>
  21 #include <boost/utility/typed_in_place_factory.hpp>
  22
  23 #include <runtime/vm/translator/abi-x64.h>
  24
  25 /*
  26  * Please don't include this unless your file implements methods of
  27  * TranslatorX64; you won't like it. It pollutes the namespace, makes
  28  * "KindOfString" #error, makes your TRACEMOD tx64, and tortures a kitten.
  29  */
  30
  31 using namespace HPHP::VM::Transl::reg;
  32 using namespace HPHP::Util;
  33 using namespace HPHP::Trace;
  34 using std::max;
  35
  36 namespace HPHP {
  37 namespace VM {
  38 namespace Transl {
  39
  40 static const Trace::Module TRACEMOD = Trace::tx64;
  41 static const DataType BitwiseKindOfString = KindOfString;
  42
  43 #define KindOfString \
  44 #error You probably do not mean to use KindOfString in this file.
  45
  46 // RAII aids to machine code.
  47
  48 template<int StackParity>
  49 class PhysRegSaverParity {
  50 protected:
  51   X64Assembler& a;
  52   RegSet s;
  53   int numElts;
  54 public:
  55   PhysRegSaverParity(X64Assembler& a_, RegSet s_) : a(a_), s(s_) {
  56     RegSet sCopy = s;
  57     numElts = 0;
  58     PhysReg reg;
  59     while (sCopy.findFirst(reg)) {
  60       a.   pushr(reg);
  61       sCopy.remove(reg);
  62       numElts++;
  63     }
  64     if ((numElts & 1) == StackParity) {
  65       // Maintain stack evenness for SIMD compatibility.
  66       a.   sub_imm32_reg64(8, rsp);
  67     }
  68   }
  69
  70   ~PhysRegSaverParity() {
  71     if ((numElts & 1) == StackParity) {
  72       // See above; stack parity.
  73       a.   add_imm32_reg64(8, rsp);
  74     }
  75     RegSet sCopy = s;
  76     PhysReg reg;
  77     while (sCopy.findLast(reg)) {
  78       a.   popr(reg);
  79       sCopy.remove(reg);
  80     }
  81   }
  82 };
  83
  84 // In shared stubs, we've already made the stack odd by calling
  85 // from a to astubs. Calls from a are on an even rsp.
  86 typedef PhysRegSaverParity<0> PhysRegSaverStub;
  87 typedef PhysRegSaverParity<1> PhysRegSaver;
  88
  89 // Put FreezeRegs in a scope around any emit calls where the caller needs
  90 // to be sure the callee will not modify the state of the register
  91 // map.  (Arises in situations with conditional code often.)
  92 struct FreezeRegs : private boost::noncopyable {
  93   explicit FreezeRegs(RegAlloc& regs) : m_regs(regs) { m_regs.freeze(); }
  94   ~FreezeRegs() { m_regs.defrost(); }
  95
  96 private:
  97   RegAlloc& m_regs;
  98 };
  99
 100 class RedirectSpillFill : boost::noncopyable {
 101   X64Assembler* const m_oldSpf;
 102 public:
 103   explicit RedirectSpillFill(X64Assembler* newCode)
 104     : m_oldSpf(tx64->m_spillFillCode)
 105   {
 106     tx64->m_spillFillCode = newCode;
 107   }
 108   ~RedirectSpillFill() {
 109     tx64->m_spillFillCode = m_oldSpf;
 110   }
 111 };
 112
 113 // DiamondGuard is a scoped way to protect register allocator state around
 114 // control flow. When we enter some optional code that may affect the state
 115 // of the register file, we copy the register file's state, and redirect any
 116 // spills and fills to the branch's body.
 117 //
 118 // When we're ready to rejoin the main control flow, we bring the registers
 119 // back into the state they were in, and restore the old spill/fill
 120 // destinations.
 121 class DiamondGuard : boost::noncopyable {
 122   RedirectSpillFill m_spfChanger;
 123 public:
 124   explicit DiamondGuard(X64Assembler& a) : m_spfChanger(&a) {
 125     tx64->m_savedRegMaps.push(
 126       TranslatorX64::SavedRegState(this, tx64->m_regMap));
 127   }
 128   ~DiamondGuard() {
 129     ASSERT(!tx64->m_savedRegMaps.empty());
 130     ASSERT(tx64->m_savedRegMaps.top().saver == this);
 131
 132     // Bring the register state back to its state in the main body.
 133     //
 134     // Note: it's important that tx64->m_regMap is the branch
 135     // RegAlloc during this.  See RegAlloc::reconcile.
 136     tx64->m_savedRegMaps.top().savedState.reconcile(tx64->m_regMap);
 137     tx64->m_regMap = tx64->m_savedRegMaps.top().savedState;
 138     tx64->m_savedRegMaps.pop();
 139   }
 140 };
 141
 142 // Helper for use with UnlikelyIfBlock when you have a complex else
 143 // branch that needs to make changes to the register file.
 144 //
 145 // For an example usage see UnlikelyIfBlock.
 146 class DiamondReturn : boost::noncopyable {
 147   X64Assembler* m_branchA;
 148   X64Assembler* m_mainA;
 149   TCA m_branchJmp;
 150   TCA m_finishBranchFrontier;
 151
 152 private:
 153   template<int> friend class UnlikelyIfBlock;
 154
 155   void initBranch(X64Assembler* branchA, X64Assembler* mainA) {
 156     /*
 157      * DiamondReturn must be used with branches going to different
 158      * code regions.
 159      */
 160     ASSERT(branchA != mainA);
 161
 162     m_branchA = branchA;
 163     m_mainA = mainA;
 164     tx64->m_savedRegMaps.push(
 165       TranslatorX64::SavedRegState(this, tx64->m_regMap));
 166   }
 167
 168   void finishBranch(TCA jmp, TCA frontier) {
 169     m_branchJmp = jmp;
 170     m_finishBranchFrontier = frontier;
 171
 172     // If there's some reason to do something other than this we have
 173     // to change the way this class works.
 174     const int UNUSED kJumpSize = 5;
 175     ASSERT(m_finishBranchFrontier == m_branchJmp + kJumpSize);
 176
 177     // We're done with the branch, so save the branch's state and
 178     // switch back to the main line's state.
 179     swapRegMaps();
 180   }
 181
 182   bool finishedBranch() const { return m_branchJmp != 0; }
 183
 184   void swapRegMaps() {
 185     ASSERT(!tx64->m_savedRegMaps.empty());
 186     ASSERT(tx64->m_savedRegMaps.top().saver == this);
 187     std::swap(tx64->m_savedRegMaps.top().savedState, tx64->m_regMap);
 188   }
 189
 190   void emitReconciliation() {
 191     ASSERT(!tx64->m_savedRegMaps.empty());
 192     ASSERT(tx64->m_savedRegMaps.top().saver == this);
 193
 194     RedirectSpillFill spfRedir(m_branchA);
 195
 196     if (finishedBranch()) {
 197       // We need tx64->m_regMap to point at the branch during
 198       // reconciliation.  See RegAlloc::reconcile().
 199       swapRegMaps();
 200     }
 201     RegAlloc& branchState = tx64->m_regMap;
 202     RegAlloc& currentState = tx64->m_savedRegMaps.top().savedState;
 203     currentState.reconcile(branchState);
 204     tx64->m_regMap = currentState;
 205     tx64->m_savedRegMaps.pop();
 206   }
 207
 208 public:
 209   explicit DiamondReturn()
 210     : m_branchA(0)
 211     , m_branchJmp(0)
 212   {}
 213
 214   void kill() {
 215     m_mainA = NULL;
 216   }
 217
 218   ~DiamondReturn() {
 219     ASSERT(m_branchA &&
 220       "DiamondReturn was created without being passed to UnlikelyIfBlock");
 221
 222     if (!m_mainA) {
 223       /*
 224        * We were killed. eg the UnlikelyIfBlock took a side exit, so
 225        * no reconciliation/branch back to a is required.
 226        */
 227       return;
 228     }
 229
 230     if (!finishedBranch()) {
 231       /*
 232        * In this case, we're reconciling the branch even though it
 233        * isn't really finished (no one ever called finishBranch()), so
 234        * we just need to emit spills/fills now and not be as clever as
 235        * below.  See UnlikelyIfBlock::reconcileEarly.
 236        */
 237       emitReconciliation();
 238       return;
 239     }
 240
 241     const TCA currentBranchFrontier = m_branchA->code.frontier;
 242     const bool branchFrontierMoved =
 243       currentBranchFrontier != m_finishBranchFrontier;
 244
 245     /*
 246      * If the branch frontier hasn't moved since the branch was
 247      * finished, we don't need the jmp that was already emitted
 248      * anymore, so rewind so we can potentially overwrite it with
 249      * spills/fills.
 250      */
 251     if (!branchFrontierMoved) {
 252       m_branchA->code.frontier = m_branchJmp;
 253     }
 254
 255     // Send out reconciliation code to the branch area.  We want to
 256     // bring the state of the branch's register file into sync with
 257     // the main-line.
 258     const TCA spfStart = m_branchA->code.frontier;
 259     emitReconciliation();
 260     const bool hadAnySpf = spfStart != m_branchA->code.frontier;
 261
 262     if (branchFrontierMoved) {
 263       /*
 264        * In this case, more than one DiamondReturn is being used and
 265        * there are multiple unlikely branches.
 266        *
 267        * If there was no reconciliation code it's not big deal, we'll
 268        * just patch the existing jmp to go to the return in m_mainA.
 269        * But if we needed reconciliation code, we'll instead want to
 270        * patch it to jump there.
 271        */
 272       if (hadAnySpf) {
 273         m_branchA->patchJmp(m_branchJmp, spfStart);
 274         m_branchA->jmp(m_mainA->code.frontier);
 275       } else {
 276         m_branchA->patchJmp(m_branchJmp, m_mainA->code.frontier);
 277       }
 278     } else {
 279       ASSERT(spfStart == m_branchJmp);
 280       m_branchA->jmp(m_mainA->code.frontier);
 281     }
 282   }
 283 };
 284
 285 // Code to profile how often our UnlikelyIfBlock branches are taken in
 286 // practice. Enable with TRACE=unlikely:1
 287 struct UnlikelyHitRate {
 288   litstr key;
 289   uint64_t check;
 290   uint64_t hit;
 291   UnlikelyHitRate() : key(nullptr), check(0), hit(0) {}
 292
 293   float rate() const {
 294     return 100.0 * hit / check;
 295   }
 296   bool operator<(const UnlikelyHitRate& b) const {
 297     return rate() > b.rate();
 298   }
 299 };
 300 typedef hphp_hash_map<litstr, UnlikelyHitRate, pointer_hash<const char>>
 301   UnlikelyHitMap;
 302 extern __thread UnlikelyHitMap* tl_unlikelyHits;
 303
 304 static void recordUnlikelyProfile(litstr key, int64 hit) {
 305   UnlikelyHitRate& r = (*tl_unlikelyHits)[key];
 306   r.key = key;
 307   if (hit) {
 308     r.hit++;
 309   } else {
 310     r.check++;
 311   }
 312 }
 313
 314 inline void emitUnlikelyProfile(bool hit, bool saveFlags,
 315                                 X64Assembler& a) {
 316   if (!Trace::moduleEnabledRelease(Trace::unlikely)) return;
 317   const ssize_t sz = 1024;
 318   char key[sz];
 319
 320   // Clean up filename
 321   std::string file =
 322     boost::filesystem::path(tx64->m_curFile).filename().string();
 323
 324   // Get instruction if wanted
 325   const NormalizedInstruction* ni = tx64->m_curNI;
 326   std::string inst;
 327   if (Trace::moduleEnabledRelease(Trace::unlikely, 2)) {
 328     inst = std::string(", ") + (ni ? opcodeToName(ni->op()) : "<none>");
 329   }
 330   const char* fmt = Trace::moduleEnabledRelease(Trace::unlikely, 3) ?
 331     "%-25s:%-5d, %-28s%s" :
 332     "%-25s:%-5d (%-28s%s)";
 333   if (snprintf(key, sz, fmt,
 334                file.c_str(), tx64->m_curLine, tx64->m_curFunc,
 335                inst.c_str()) >= sz) {
 336     key[sz-1] = '\0';
 337   }
 338   litstr data = StringData::GetStaticString(key)->data();
 339
 340   if (saveFlags) a.pushf();
 341   {
 342     PhysRegSaver regs(a, kAllX64Regs);
 343     int i = 0;
 344     a.emitImmReg((intptr_t)data, argNumToRegName[i++]);
 345     a.emitImmReg((intptr_t)hit, argNumToRegName[i++]);
 346     a.call((TCA)recordUnlikelyProfile);
 347   }
 348   if (saveFlags) a.popf();
 349 }
 350
 351 inline void initUnlikelyProfile() {
 352   if (!Trace::moduleEnabledRelease(Trace::unlikely)) return;
 353   tl_unlikelyHits = new UnlikelyHitMap();
 354 }
 355
 356 inline void dumpUnlikelyProfile() {
 357   if (!Trace::moduleEnabledRelease(Trace::unlikely)) return;
 358   std::vector<UnlikelyHitRate> hits;
 359   UnlikelyHitRate overall;
 360   overall.key = "total";
 361   for (auto item : *tl_unlikelyHits) {
 362     overall.check += item.second.check;
 363     overall.hit += item.second.hit;
 364     hits.push_back(item.second);
 365   }
 366   if (hits.empty()) return;
 367   auto cmp = [&](const UnlikelyHitRate& a, const UnlikelyHitRate& b) {
 368     return a.hit > b.hit ? true : a.hit == b.hit ? a.check > b.check : false;
 369   };
 370   std::sort(hits.begin(), hits.end(), cmp);
 371   Trace::traceRelease("UnlikelyIfBlock hit rates for %s:\n",
 372                       g_context->getRequestUrl(50).c_str());
 373   const char* fmt = Trace::moduleEnabledRelease(Trace::unlikely, 3) ?
 374     "%6.2f, %8llu, %8llu, %5.1f, %s\n" :
 375     "%6.2f%% (%8llu / %8llu, %5.1f%% of total): %s\n";
 376   auto printRate = [&](const UnlikelyHitRate& hr) {
 377     Trace::traceRelease(fmt,
 378                         hr.rate(), hr.hit, hr.check, hr.key,
 379                         100.0 * hr.hit / overall.hit);
 380   };
 381   printRate(overall);
 382   std::for_each(hits.begin(), hits.end(), printRate);
 383   Trace::traceRelease("\n");
 384
 385   delete tl_unlikelyHits;
 386   tl_unlikelyHits = nullptr;
 387 }
 388
 389 // UnlikelyIfBlock:
 390 //
 391 //  Branch to distant code (that we presumably don't expect to
 392 //  take). This helps keep hot paths compact.
 393 //
 394 //  A common pattern using this involves patching the jump in astubs
 395 //  to jump past the normal control flow in a (as in the following
 396 //  example).  Do this using DiamondReturn so the register allocator
 397 //  state will be properly maintained.  (Spills/fills to keep the
 398 //  states in sync will be emitted on the unlikely path.)
 399 //
 400 // Example:
 401 //
 402 //  {
 403 //    PhysReg inputParam = i.getReg(i.inputs[0]->location);
 404 //    a.   test_reg_reg(inputParam, inputParam);
 405 //    DiamondReturn retFromStubs;
 406 //    {
 407 //      UnlikelyIfBlock<CC_Z> ifNotRax(a, astubs, &retFromStubs);
 408 //      EMIT_CALL(a, TCA(launch_nuclear_missiles));
 409 //    }
 410 //    // The inputParam was non-zero, here is the likely branch:
 411 //    m_regMap.allocOutputRegs(i);
 412 //    emitMovRegReg(inputParam, m_regMap.getReg(i.outLocal->location));
 413 //    // ~DiamondReturn patches the jump, and reconciles the branch
 414 //    // with the main line.  (In this case it will fill the outLocal
 415 //    // register since the main line thinks it is dirty.)
 416 //  }
 417 //  // The two cases are joined here.  We can do logic that was
 418 //  // independent of whether the branch was taken, if necessary.
 419 //  emitMovRegReg(i.outLocal, m_regMap.getReg(i.outStack->location));
 420 //
 421 // Note: it is ok to nest UnlikelyIfBlocks, as long as their
 422 // corresponding DiamondReturns are correctly destroyed in reverse
 423 // order.  But also note that this can lead to more jumps on the
 424 // unlikely branch (see ~DiamondReturn).
 425 template <int Jcc>
 426 struct UnlikelyIfBlock {
 427   X64Assembler& m_likely;
 428   X64Assembler& m_unlikely;
 429   TCA m_likelyPostBranch;
 430
 431   DiamondReturn* m_returnDiamond;
 432   bool m_externalDiamond;
 433   boost::optional<FreezeRegs> m_ice;
 434
 435   explicit UnlikelyIfBlock(X64Assembler& likely,
 436                            X64Assembler& unlikely,
 437                            DiamondReturn* returnDiamond = 0)
 438     : m_likely(likely)
 439     , m_unlikely(unlikely)
 440     , m_returnDiamond(returnDiamond ? returnDiamond : new DiamondReturn())
 441     , m_externalDiamond(!!returnDiamond)
 442   {
 443     emitUnlikelyProfile(false, true, m_likely);
 444     m_likely.jcc(Jcc, m_unlikely.code.frontier);
 445     emitUnlikelyProfile(true, false, m_unlikely);
 446     m_likelyPostBranch = m_likely.code.frontier;
 447     m_returnDiamond->initBranch(&unlikely, &likely);
 448     tx64->m_spillFillCode = &unlikely;
 449   }
 450
 451   ~UnlikelyIfBlock() {
 452     TCA jmpAddr = m_unlikely.code.frontier;
 453     m_unlikely.jmp(m_likelyPostBranch);
 454     if (m_returnDiamond) {
 455       m_returnDiamond->finishBranch(jmpAddr, m_unlikely.code.frontier);
 456       if (!m_externalDiamond) {
 457         delete m_returnDiamond;
 458       }
 459     }
 460     tx64->m_spillFillCode = &m_likely;
 461   }
 462
 463   /*
 464    * Force early reconciliation between the branch and main line.
 465    * Using this has some tricky cases, part of which is that you can't
 466    * allocate registers anymore in the branch if you do this (so we'll
 467    * freeze until ~UnlikelyIfBlock).
 468    *
 469    * It's also almost certainly error if we have m_externalDiamond, so
 470    * for now that's an assert.
 471    */
 472   void reconcileEarly() {
 473     ASSERT(!m_externalDiamond);
 474     delete m_returnDiamond;
 475     m_returnDiamond = 0;
 476     m_ice = boost::in_place<FreezeRegs>(boost::ref(tx64->m_regMap));
 477   }
 478 };
 479
 480 #define UnlikelyIfBlock                                                 \
 481   m_curFile = __FILE__; m_curFunc = __FUNCTION__; m_curLine = __LINE__; \
 482   UnlikelyIfBlock
 483
 484 // A CondBlock is an RAII structure for emitting conditional code. It
 485 // compares the source register at fieldOffset with fieldValue, and
 486 // conditionally branches over the enclosing block of assembly on the
 487 // passed-in condition-code.
 488 //
 489 //  E.g.:
 490 //    {
 491 //      RefCountedOnly ifRefCounted(a, rdi, 0);
 492 //      emitIncRef(rdi);
 493 //    }
 494 //
 495 // will only execute emitIncRef if we find at runtime that rdi points at
 496 // a ref-counted cell.
 497 //
 498 // It's ok to do reconcilable register operations in the body.
 499 template<int FieldOffset, int FieldValue, int Jcc>
 500 struct CondBlock {
 501   X64Assembler& m_a;
 502   int m_off;
 503   TCA m_jcc8;
 504   DiamondGuard* m_dg;
 505
 506   CondBlock(X64Assembler& a, PhysReg reg, int offset = 0)
 507       : m_a(a), m_off(offset), m_dg(new DiamondGuard(a)) {
 508     int typeDisp = m_off + FieldOffset;
 509     a.   cmp_imm32_disp_reg32(FieldValue, typeDisp, reg);
 510     m_jcc8 = a.code.frontier;
 511     a.   jcc8(Jcc, m_jcc8);
 512     // ...
 513   }
 514
 515   ~CondBlock() {
 516     delete m_dg;
 517     m_a.patchJcc8(m_jcc8, m_a.code.frontier);
 518   }
 519 };
 520
 521 // IfRefCounted --
 522 //   Emits if (IS_REFCOUNTED_TYPE()) { ... }
 523 typedef CondBlock <TVOFF(m_type),
 524                    KindOfRefCountThreshold,
 525                    CC_LE> IfRefCounted;
 526
 527 typedef CondBlock <TVOFF(m_type),
 528                    KindOfRef,
 529                    CC_NZ> IfVariant;
 530
 531 typedef CondBlock <TVOFF(m_type),
 532                    KindOfUninit,
 533                    CC_Z> UnlessUninit;
 534
 535 /*
 536  * locToRegDisp --
 537  *
 538  * Helper code for stack frames. The struct is a "template" in the
 539  * non-C++ sense: we don't build source-level stack frames in C++
 540  * for the most part, but its offsets tell us where to find fields
 541  * in assembly.
 542  *
 543  * If we were physically pushing stack frames, we would push them
 544  * in reverse order to what you see here.
 545  */
 546 static void
 547 locToRegDisp(const Location& l, PhysReg *outbase, int *outdisp,
 548              const Func* f = NULL) {
 549   assert_not_implemented((l.space == Location::Stack ||
 550                           l.space == Location::Local ||
 551                           l.space == Location::Iter));
 552   *outdisp = cellsToBytes(Translator::locPhysicalOffset(l, f));
 553   *outbase = l.space == Location::Stack ? rVmSp : rVmFp;
 554 }
 555
 556 // Common code emission patterns.
 557
 558 // emitStoreImm --
 559 //   Try to use a nice encoding for the size and value.
 560 static void
 561 emitStoreImm(X64Assembler& a, uint64_t imm, PhysReg r, int off,
 562              int size = sz::qword, RegAlloc* regAlloc = NULL) {
 563   if (size == sz::qword) {
 564     PhysReg immReg = regAlloc ? regAlloc->getImmReg(imm) : InvalidReg;
 565     if (immReg == InvalidReg) {
 566       emitImmReg(a, imm, rScratch);
 567       immReg = rScratch;
 568     }
 569     a.   store_reg64_disp_reg64(immReg, off, r);
 570   } else if (size == sz::dword) {
 571     a.   store_imm32_disp_reg(imm, off, r);
 572   } else {
 573     not_implemented();
 574   }
 575 }
 576
 577 // vstackOffset --
 578 // emitVStackStore --
 579 //
 580 //   Store to the virtual stack at a normalized instruction boundary.
 581 //   Since rVmSp points to the stack at entry to the current BB, we need to
 582 //   adjust stack references relative to it.
 583 //
 584 //   For all parameters, offsets and sizes are in bytes. Sizes are expected
 585 //   to be a hardware size: 1, 2, 4, or 8 bytes.
 586 static inline int
 587 vstackOffset(const NormalizedInstruction& ni, COff off) {
 588   return off - cellsToBytes(ni.stackOff);
 589 }
 590
 591 static inline void
 592 emitVStackStoreImm(X64Assembler &a, const NormalizedInstruction &ni,
 593                    uint64_t imm, int off, int size = sz::qword,
 594                    RegAlloc *regAlloc = NULL) {
 595   int hwOff = vstackOffset(ni, off);
 596   emitStoreImm(a, imm, rVmSp, hwOff, size, regAlloc);
 597 }
 598
 599 static inline void
 600 emitVStackStore(X64Assembler &a, const NormalizedInstruction &ni,
 601                 PhysReg src, int off, int size = sz::qword) {
 602   int hwOff = vstackOffset(ni, off);
 603   if (size == sz::qword) {
 604     a.    store_reg64_disp_reg64(src, hwOff, rVmSp);
 605   } else if (size == sz::dword) {
 606     a.    store_reg32_disp_reg64(src, hwOff, rVmSp);
 607   } else {
 608     not_implemented();
 609   }
 610 }
 611
 612 static inline const StringData*
 613 local_name(const Location& l) {
 614   ASSERT(l.isLocal());
 615   const StringData* ret = curFunc()->localNames()[l.offset];
 616   ASSERT(ret->isStatic());
 617   return ret;
 618 }
 619
 620 // emitDispDeref --
 621 // emitDeref --
 622 // emitStoreTypedValue --
 623 // emitStoreUninitNull --
 624 // emitStoreNull --
 625 //
 626 //   Helpers for common cell operations.
 627 //
 628 //   Dereference the var in the cell whose address lives in src into
 629 //   dest.
 630 static void
 631 emitDispDeref(X64Assembler &a, PhysReg src, int disp,
 632               PhysReg dest) {
 633   a.    load_reg64_disp_reg64(src, disp + TVOFF(m_data), dest);
 634 }
 635
 636 static void
 637 emitDeref(X64Assembler &a, PhysReg src, PhysReg dest) {
 638   emitDispDeref(a, src, 0, dest);
 639 }
 640
 641 static void
 642 emitDerefIfVariant(X64Assembler &a, PhysReg reg) {
 643   if (RuntimeOption::EvalJitCmovVarDeref) {
 644     a.cmp_imm32_disp_reg32(KindOfRef, TVOFF(m_type), reg);
 645     a.cload_reg64_disp_reg64(CC_Z, reg, TVOFF(m_data), reg);
 646   } else {
 647     IfVariant ifVar(a, reg);
 648     emitDeref(a, reg, reg);
 649   }
 650 }
 651
 652 // NB: leaves count field unmodified. Does not store to m_data if type
 653 // is a null type.
 654 static inline void
 655 emitStoreTypedValue(X64Assembler& a, DataType type, PhysReg val,
 656                     int disp, PhysReg dest, bool writeType = true) {
 657   if (writeType) {
 658     a.  store_imm32_disp_reg(type, disp + TVOFF(m_type), dest);
 659   }
 660   if (!IS_NULL_TYPE(type)) {
 661     ASSERT(val != reg::noreg);
 662     a.  store_reg64_disp_reg64(val, disp + TVOFF(m_data), dest);
 663   }
 664 }
 665
 666 static inline void
 667 emitStoreInvalid(X64Assembler& a, int disp, PhysReg dest) {
 668   a.    store_imm64_disp_reg64(0xfacefacefacefaceULL, disp + TVOFF(m_data),
 669                                dest);
 670   a.    store_imm32_disp_reg(0xfacefaceU, disp + TVOFF(_count), dest);
 671   a.    store_imm32_disp_reg(KindOfInvalid, disp + TVOFF(m_type), dest);
 672 }
 673
 674 static inline void
 675 emitStoreUninitNull(X64Assembler& a,
 676                     int disp,
 677                     PhysReg dest) {
 678   // OK to leave garbage in m_data, _count.
 679   a.    store_imm32_disp_reg(KindOfUninit, disp + TVOFF(m_type), dest);
 680 }
 681
 682 static inline void
 683 emitStoreNull(X64Assembler& a,
 684               int disp,
 685               PhysReg dest) {
 686   a.    store_imm32_disp_reg(KindOfNull, disp + TVOFF(m_type), dest);
 687   // It's ok to leave garbage in m_data, _count for KindOfNull.
 688 }
 689
 690 static inline void
 691 emitStoreNull(X64Assembler& a, const Location& where) {
 692   PhysReg base;
 693   int disp;
 694   locToRegDisp(where, &base, &disp);
 695   emitStoreNull(a, disp, base);
 696 }
 697
 698 /*
 699  * The 'zero' argument can be noreg, rFlag, or a normal register
 700  * name. If it's noreg, a 0 immediate will be stored to _count. If
 701  * it's rFlag, nothing will be stored to _count. If it's a normal
 702  * register name, the contents of that register (hopefully set to zero
 703  * by the caller) are stored to _count.
 704  */
 705 static inline void
 706 emitCopyTo(X64Assembler& a,
 707            PhysReg src,
 708            int srcOff,
 709            PhysReg dest,
 710            int destOff,
 711            PhysReg scratch) {
 712   ASSERT(src != scratch);
 713   // This is roughly how gcc compiles this.
 714   a.    load_reg64_disp_reg64(src, srcOff + TVOFF(m_data), scratch);
 715   // Blow off _count.
 716   a.    store_reg64_disp_reg64(scratch, destOff + TVOFF(m_data), dest);
 717   a.    load_reg64_disp_reg32(src, srcOff + TVOFF(m_type), scratch);
 718   a.    store_reg32_disp_reg64(scratch, destOff + TVOFF(m_type), dest);
 719 }
 720
 721 // ArgManager -- support for passing VM-level data to helper functions.
 722 class ArgManager {
 723   typedef HPHP::VM::Transl::X64Assembler& A;
 724 public:
 725   ArgManager(TranslatorX64 &tx64, A& a) : m_tx64(tx64), m_a(a) { }
 726
 727   void addImm(uint64_t imm) {
 728     TRACE(6, "ArgManager: push arg %zd imm:%lu\n",
 729           m_args.size(), imm);
 730     m_args.push_back(ArgContent(ArgContent::ArgImm, InvalidReg, imm));
 731   }
 732
 733   void addLoc(const Location &loc) {
 734     TRACE(6, "ArgManager: push arg %zd loc:(%s, %lld)\n",
 735           m_args.size(), loc.spaceName(), loc.offset);
 736     m_args.push_back(ArgContent(ArgContent::ArgLoc, loc));
 737   }
 738
 739   void addDeref(const Location &loc) {
 740     TRACE(6, "ArgManager: push arg %zd deref:(%s, %lld)\n",
 741           m_args.size(), loc.spaceName(), loc.offset);
 742     m_args.push_back(ArgContent(ArgContent::ArgDeref, loc));
 743   }
 744
 745   void addReg(PhysReg reg) {
 746     TRACE(6, "ArgManager: push arg %zd reg:r%d\n",
 747           m_args.size(), reg);
 748     m_args.push_back(ArgContent(ArgContent::ArgReg, reg, 0));
 749   }
 750
 751   void addRegPlus(PhysReg reg, int32_t off) {
 752     TRACE(6, "ArgManager: push arg %zd regplus:r%d+%d\n",
 753           m_args.size(), reg, off);
 754     m_args.push_back(ArgContent(ArgContent::ArgRegPlus, reg, off));
 755   }
 756
 757   void addLocAddr(const Location &loc) {
 758     TRACE(6, "ArgManager: push arg %zd addr:(%s, %lld)\n",
 759           m_args.size(), loc.spaceName(), loc.offset);
 760     ASSERT(!loc.isLiteral());
 761     m_args.push_back(ArgContent(ArgContent::ArgLocAddr, loc));
 762   }
 763
 764   void emitArguments() {
 765     size_t n = m_args.size();
 766     ASSERT((int)n <= kNumRegisterArgs);
 767     cleanLocs();
 768     std::map<PhysReg, size_t> used;
 769     std::vector<PhysReg> actual(n, InvalidReg);
 770     computeUsed(used, actual);
 771     shuffleRegisters(used, actual);
 772     emitValues(actual);
 773   }
 774
 775 private:
 776   struct ArgContent {
 777     enum ArgKind {
 778       ArgImm, ArgLoc, ArgDeref, ArgReg, ArgRegPlus, ArgLocAddr
 779     } m_kind;
 780     PhysReg m_reg;
 781     const Location *m_loc;
 782     uint64_t m_imm;
 783
 784     ArgContent(ArgKind kind, PhysReg reg, uint64_t imm) :
 785       m_kind(kind), m_reg(reg), m_loc(NULL), m_imm(imm) { }
 786     ArgContent(ArgKind kind, const Location &loc) :
 787       m_kind(kind), m_reg(InvalidReg), m_loc(&loc), m_imm(0) { }
 788   };
 789
 790   TranslatorX64& m_tx64;
 791   A& m_a;
 792   std::vector<ArgContent> m_args;
 793
 794   ArgManager(); // Don't build without reference to translator
 795
 796   void cleanLocs();
 797   void computeUsed(std::map<PhysReg, size_t> &used,
 798                    std::vector<PhysReg> &actual);
 799   void shuffleRegisters(std::map<PhysReg, size_t> &used,
 800                      std::vector<PhysReg> &actual);
 801   void emitValues(std::vector<PhysReg> &actual);
 802 };
 803
 804 // Some macros to make writing calls palatable. You have to "type" the
 805 // arguments
 806 #define IMM(i)       _am.addImm(i)
 807 #define V(loc)       _am.addLoc(loc)
 808 #define DEREF(loc)   _am.addDeref(loc)
 809 #define R(r)         _am.addReg(r)
 810 #define RPLUS(r,off) _am.addRegPlus(r, off)
 811 #define A(loc)       _am.addLocAddr(loc)
 812 #define IE(cond, argIf, argElse) \
 813   ((cond) ? (argIf) : (argElse))
 814 static inline void voidFunc() {}
 815 #define ID(argDbg) IE(debug, (argDbg), voidFunc())
 816
 817 #define EMIT_CALL_PROLOGUE(a) do { \
 818   SpaceRecorder sr("_HCallInclusive", a);  \
 819   ArgManager _am(*this, a);       \
 820   prepareCallSaveRegs();
 821
 822 #define EMIT_CALL_EPILOGUE(a, dest) \
 823   _am.emitArguments();           \
 824   { \
 825     SpaceRecorder sr("_HCallExclusive", a); \
 826     emitCall(a, (TCA)(dest), true);         \
 827   } \
 828 } while(0)
 829
 830 #define EMIT_CALL(a, dest, ...) \
 831   EMIT_CALL_PROLOGUE(a)         \
 832   __VA_ARGS__ ;                 \
 833   EMIT_CALL_EPILOGUE(a, dest)
 834
 835 #define EMIT_RCALL(a, ni, dest, ...) \
 836   EMIT_CALL(a, dest, __VA_ARGS__);   \
 837   recordReentrantCall(a, ni);
 838
 839 // typeReentersOnRelease --
 840 //   Returns whether the release helper for a given type can
 841 //   reenter.
 842 static bool typeReentersOnRelease(DataType type) {
 843   return IS_REFCOUNTED_TYPE(type) && type != BitwiseKindOfString;
 844 }
 845
 846 // supportedPlan --
 847 // nativePlan --
 848 // simplePlan --
 849 //   Some helpers for analyze* methods.
 850 static inline TXFlags
 851 plan(bool cond, TXFlags successFlags, TXFlags fallbackFlags=Interp) {
 852   return cond ? successFlags : fallbackFlags;
 853 }
 854
 855 static inline TXFlags simplePlan(bool cond) { return plan(cond, Simple); }
 856 static inline TXFlags simpleOrSupportedPlan(bool cond) {
 857   return plan(cond, Simple, Supported);
 858 }
 859 static inline TXFlags supportedPlan(bool cond) { return plan(cond, Supported); }
 860 static inline TXFlags nativePlan(bool cond) { return plan(cond, Native); }
 861
 862 static inline TXFlags planHingesOnRefcounting(DataType type) {
 863   return !IS_REFCOUNTED_TYPE(type) ? Native :
 864          !typeReentersOnRelease(type) ? Simple :
 865          Supported;
 866 }
 867
 868 static inline const char* getContextName() {
 869   Class* ctx = arGetContextClass(curFrame());
 870   return ctx ? ctx->name()->data() : ":anonymous:";
 871 }
 872
 873 template<class T>
 874 struct Nuller : private boost::noncopyable {
 875   explicit Nuller(const T** p) : p(p) {}
 876   ~Nuller() { *p = 0; }
 877   T const** const p;
 878 };
 879
 880 template<class T>
 881 struct Deleter : private boost::noncopyable {
 882   explicit Deleter(T** p) : p(p) {}
 883   ~Deleter() {
 884     delete *p;
 885     *p = NULL;
 886   }
 887   T** p;
 888 };
 889
 890 }}}
 891
 892 #endif
 893