hphp/runtime/vm/jit/unique-stubs.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/unique-stubs.h"
  18
  19 #include "hphp/runtime/base/arch.h"
  20 #include "hphp/runtime/base/datatype.h"
  21 #include "hphp/runtime/base/rds-header.h"
  22 #include "hphp/runtime/base/runtime-option.h"
  23 #include "hphp/runtime/base/tv-helpers.h"
  24 #include "hphp/runtime/vm/bytecode.h"
  25 #include "hphp/runtime/vm/debug/debug.h"
  26 #include "hphp/runtime/vm/hhbc.h"
  27 #include "hphp/runtime/vm/srckey.h"
  28 #include "hphp/runtime/vm/vm-regs.h"
  29
  30 #include "hphp/runtime/vm/jit/types.h"
  31 #include "hphp/runtime/vm/jit/abi.h"
  32 #include "hphp/runtime/vm/jit/align.h"
  33 #include "hphp/runtime/vm/jit/code-cache.h"
  34 #include "hphp/runtime/vm/jit/code-gen-cf.h"
  35 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  36 #include "hphp/runtime/vm/jit/fixup.h"
  37 #include "hphp/runtime/vm/jit/mc-generator.h"
  38 #include "hphp/runtime/vm/jit/phys-reg.h"
  39 #include "hphp/runtime/vm/jit/phys-reg-saver.h"
  40 #include "hphp/runtime/vm/jit/service-requests.h"
  41 #include "hphp/runtime/vm/jit/smashable-instr.h"
  42 #include "hphp/runtime/vm/jit/stack-offsets.h"
  43 #include "hphp/runtime/vm/jit/stack-overflow.h"
  44 #include "hphp/runtime/vm/jit/translator-inline.h"
  45 #include "hphp/runtime/vm/jit/unique-stubs-arm.h"
  46 #include "hphp/runtime/vm/jit/unique-stubs-x64.h"
  47 #include "hphp/runtime/vm/jit/unwind-itanium.h"
  48 #include "hphp/runtime/vm/jit/vasm-gen.h"
  49 #include "hphp/runtime/vm/jit/vasm-instr.h"
  50 #include "hphp/runtime/vm/jit/vasm-reg.h"
  51
  52 #include "hphp/runtime/ext/asio/asio-blockable.h"
  53 #include "hphp/runtime/ext/asio/ext_async-function-wait-handle.h"
  54 #include "hphp/runtime/ext/asio/ext_async-generator.h"
  55 #include "hphp/runtime/ext/asio/ext_wait-handle.h"
  56 #include "hphp/runtime/ext/generator/ext_generator.h"
  57
  58 #include "hphp/util/abi-cxx.h"
  59 #include "hphp/util/asm-x64.h"
  60 #include "hphp/util/data-block.h"
  61 #include "hphp/util/disasm.h"
  62 #include "hphp/util/trace.h"
  63
  64 #include <folly/Format.h>
  65
  66 #include <algorithm>
  67
  68 namespace HPHP { namespace jit {
  69
  70 ///////////////////////////////////////////////////////////////////////////////
  71
  72 TRACE_SET_MOD(ustubs);
  73
  74 ///////////////////////////////////////////////////////////////////////////////
  75
  76 namespace {
  77
  78 ///////////////////////////////////////////////////////////////////////////////
  79
  80 void alignJmpTarget(CodeBlock& cb) {
  81   align(cb, nullptr, Alignment::JmpTarget, AlignContext::Dead);
  82 }
  83
  84 void assertNativeStackAligned(Vout& v) {
  85   if (RuntimeOption::EvalHHIRGenerateAsserts) {
  86     v << call{TCA(assert_native_stack_aligned)};
  87   }
  88 }
  89
  90 void loadVMRegs(Vout& v) {
  91   v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
  92   v << load{rvmtl()[rds::kVmspOff], rvmsp()};
  93 }
  94
  95 void storeVMRegs(Vout& v) {
  96   v << store{rvmfp(), rvmtl()[rds::kVmfpOff]};
  97   v << store{rvmsp(), rvmtl()[rds::kVmspOff]};
  98 }
  99
 100 void loadMCG(Vout& v, Vreg d) {
 101   // TODO(#8060678): Why does this need to be RIP-relative?
 102   auto const imcg = reinterpret_cast<uintptr_t>(&mcg);
 103   v << loadqp{reg::rip[imcg], d};
 104 }
 105
 106 /*
 107  * Convenience wrapper around a simple vcall to `helper', with a single `arg'
 108  * and a return value in `d'.
 109  */
 110 template<class F>
 111 Vinstr simplecall(Vout& v, F helper, Vreg arg, Vreg d) {
 112   return vcall{
 113     CallSpec::direct(helper),
 114     v.makeVcallArgs({{arg}}),
 115     v.makeTuple({d}),
 116     Fixup{},
 117     DestType::SSA
 118   };
 119 }
 120
 121 ///////////////////////////////////////////////////////////////////////////////
 122
 123 TCA emitFunctionEnterHelper(CodeBlock& cb, UniqueStubs& us) {
 124   if (arch() != Arch::X64) not_implemented();
 125   return x64::emitFunctionEnterHelper(cb, us);
 126 }
 127
 128 TCA emitFreeLocalsHelpers(CodeBlock& cb, UniqueStubs& us) {
 129   if (arch() != Arch::X64) not_implemented();
 130   return x64::emitFreeLocalsHelpers(cb, us);
 131 }
 132
 133 TCA emitCallToExit(CodeBlock& cb) {
 134   if (arch() != Arch::X64) not_implemented();
 135   return x64::emitCallToExit(cb);
 136 }
 137
 138 TCA emitEndCatchHelper(CodeBlock& cb, UniqueStubs& us) {
 139   if (arch() != Arch::X64) not_implemented();
 140   return x64::emitEndCatchHelper(cb, us);
 141 }
 142
 143 ///////////////////////////////////////////////////////////////////////////////
 144
 145 TCA fcallHelper(ActRec* ar) {
 146   assert_native_stack_aligned();
 147   assertx(!ar->resumed());
 148
 149   if (LIKELY(!RuntimeOption::EvalFailJitPrologs)) {
 150     auto const tca = mcg->getFuncPrologue(
 151       const_cast<Func*>(ar->func()),
 152       ar->numArgs(),
 153       ar
 154     );
 155     if (tca) return tca;
 156   }
 157
 158   // Check for stack overflow in the same place func prologues make their
 159   // StackCheck::Early check (see irgen-func-prologue.cpp).  This handler also
 160   // cleans and syncs vmRegs for us.
 161   if (checkCalleeStackOverflow(ar)) handleStackOverflow(ar);
 162
 163   try {
 164     VMRegAnchor _(ar);
 165     if (doFCall(ar, vmpc())) {
 166       return mcg->ustubs().resumeHelperRet;
 167     }
 168     // We've been asked to skip the function body (fb_intercept).  The vmregs
 169     // have already been fixed; indicate this with a nullptr return.
 170     return nullptr;
 171   } catch (...) {
 172     // The VMRegAnchor above took care of us, but we need to tell the unwinder
 173     // (since ~VMRegAnchor() will have reset tl_regState).
 174     tl_regState = VMRegState::CLEAN;
 175     throw;
 176   }
 177 }
 178
 179 void syncFuncBodyVMRegs(ActRec* fp, void* sp) {
 180   auto& regs = vmRegsUnsafe();
 181   regs.fp = fp;
 182   regs.stack.top() = (Cell*)sp;
 183
 184   auto const nargs = fp->numArgs();
 185   auto const nparams = fp->func()->numNonVariadicParams();
 186   auto const& paramInfo = fp->func()->params();
 187
 188   auto firstDVI = InvalidAbsoluteOffset;
 189
 190   for (auto i = nargs; i < nparams; ++i) {
 191     auto const dvi = paramInfo[i].funcletOff;
 192     if (dvi != InvalidAbsoluteOffset) {
 193       firstDVI = dvi;
 194       break;
 195     }
 196   }
 197   if (firstDVI != InvalidAbsoluteOffset) {
 198     regs.pc = fp->m_func->unit()->entry() + firstDVI;
 199   } else {
 200     regs.pc = fp->m_func->getEntry();
 201   }
 202 }
 203
 204 TCA funcBodyHelper(ActRec* fp) {
 205   assert_native_stack_aligned();
 206   void* const sp = reinterpret_cast<Cell*>(fp) - fp->func()->numSlotsInFrame();
 207   syncFuncBodyVMRegs(fp, sp);
 208   tl_regState = VMRegState::CLEAN;
 209
 210   auto const func = const_cast<Func*>(fp->m_func);
 211   auto tca = mcg->getFuncBody(func);
 212   if (!tca) {
 213     tca = mcg->ustubs().resumeHelper;
 214   }
 215
 216   tl_regState = VMRegState::DIRTY;
 217   return tca;
 218 }
 219
 220 ///////////////////////////////////////////////////////////////////////////////
 221
 222 TCA emitFuncPrologueRedispatch(CodeBlock& cb) {
 223   alignJmpTarget(cb);
 224
 225   return vwrap(cb, [] (Vout& v) {
 226     auto const func = v.makeReg();
 227     v << load{rvmfp()[AROFF(m_func)], func};
 228
 229     auto const argc = v.makeReg();
 230     auto const naaf = v.makeReg();
 231     v << loadl{rvmfp()[AROFF(m_numArgsAndFlags)], naaf};
 232     v << andli{ActRec::kNumArgsMask, naaf, argc, v.makeReg()};
 233
 234     auto const nparams = v.makeReg();
 235     auto const pcounts = v.makeReg();
 236     v << loadl{func[Func::paramCountsOff()], pcounts};
 237     v << shrli{0x1, pcounts, nparams, v.makeReg()};
 238
 239     auto const sf = v.makeReg();
 240     v << cmpl{argc, nparams, sf};
 241
 242     auto const pTabOff = safe_cast<int32_t>(Func::prologueTableOff());
 243     auto const ptrSize = safe_cast<int32_t>(sizeof(LowPtr<uint8_t>));
 244
 245     // If we passed more args than declared, we might need to dispatch to the
 246     // "too many arguments" prologue.
 247     ifThen(v, CC_L, sf, [&] (Vout& v) {
 248       auto const sf = v.makeReg();
 249
 250       // If we passed fewer than kNumFixedPrologues, argc is still a valid
 251       // index into the prologue table.
 252       v << cmpli{kNumFixedPrologues, argc, sf};
 253
 254       ifThen(v, CC_NL, sf, [&] (Vout& v) {
 255         auto const dest = v.makeReg();
 256
 257         auto const nargs = v.makeReg();
 258         v << movzlq{nparams, nargs};
 259
 260         // Too many gosh-darned arguments passed.  Go to the (nparams + 1)-th
 261         // prologue, which is always the "too many args" entry point.
 262         emitLdLowPtr(v, func[nargs * ptrSize + (pTabOff + ptrSize)],
 263                      dest, sizeof(LowPtr<uint8_t>));
 264         v << jmpr{dest};
 265       });
 266     });
 267
 268     auto const nargs = v.makeReg();
 269     v << movzlq{argc, nargs};
 270
 271     auto const dest = v.makeReg();
 272     emitLdLowPtr(v, func[nargs * ptrSize + pTabOff],
 273                  dest, sizeof(LowPtr<uint8_t>));
 274     v << jmpr{dest};
 275   });
 276 }
 277
 278 TCA emitFCallHelperThunk(CodeBlock& cb) {
 279   alignJmpTarget(cb);
 280
 281   return vwrap2(cb, [] (Vout& v, Vout& vcold) {
 282     v << phplogue{rvmfp()};
 283
 284     // fcallHelper asserts native stack alignment for us.
 285     TCA (*helper)(ActRec*) = &fcallHelper;
 286     auto const dest = v.makeReg();
 287     v << simplecall(v, helper, rvmfp(), dest);
 288
 289     // Clobber rvmsp in debug builds.
 290     if (debug) v << copy{v.cns(0x1), rvmsp()};
 291
 292     auto const sf = v.makeReg();
 293     v << testq{dest, dest, sf};
 294
 295     unlikelyIfThen(v, vcold, CC_Z, sf, [&] (Vout& v) {
 296       // A nullptr dest means the callee was intercepted and should be skipped.
 297       // Make a copy of the current rvmfp(), which belongs to the callee,
 298       // before syncing VM regs.
 299       auto const callee_fp = v.makeReg();
 300       v << copy{rvmfp(), callee_fp};
 301       loadVMRegs(v);
 302
 303       // Do a PHP return to the caller---i.e., relative to the callee's frame.
 304       // Note that if intercept skips the callee, it tears down its frame but
 305       // guarantees that m_savedRip remains valid, so this is safe (and is the
 306       // only way to get the return address).
 307       //
 308       // TODO(#8908075): We've been fishing the m_savedRip out of the callee's
 309       // logically-trashed frame for a while now, but we really ought to
 310       // respect that the frame is freed and not touch it.
 311       v << phpret{callee_fp, rvmfp(), php_return_regs(), true};
 312     });
 313
 314     // Jump to the func prologue.
 315     v << tailcallphp{dest, rvmfp(), php_call_regs()};
 316   });
 317 }
 318
 319 TCA emitFuncBodyHelperThunk(CodeBlock& cb) {
 320   alignJmpTarget(cb);
 321
 322   return vwrap(cb, [] (Vout& v) {
 323     TCA (*helper)(ActRec*) = &funcBodyHelper;
 324     auto const dest = v.makeReg();
 325     v << simplecall(v, helper, rvmfp(), dest);
 326     v << jmpr{dest};
 327   });
 328 }
 329
 330 TCA emitFunctionSurprisedOrStackOverflow(CodeBlock& cb,
 331                                          const UniqueStubs& us) {
 332   alignJmpTarget(cb);
 333
 334   return vwrap(cb, [&] (Vout& v) {
 335     v << stublogue{};
 336     v << vcall{CallSpec::direct(handlePossibleStackOverflow),
 337                v.makeVcallArgs({{rvmfp()}}), v.makeTuple({})};
 338     v << tailcallstub{us.functionEnterHelper};
 339   });
 340 }
 341
 342 ///////////////////////////////////////////////////////////////////////////////
 343
 344 template<bool async>
 345 void loadGenFrame(Vout& v, Vreg d) {
 346   auto const arOff = BaseGenerator::arOff() -
 347     (async ? AsyncGenerator::objectOff() : Generator::objectOff());
 348
 349   auto const gen = v.makeReg();
 350
 351   // We have to get the Generator object from the current frame's $this, then
 352   // load the embedded frame.
 353   v << load{rvmfp()[AROFF(m_this)], gen};
 354   v << lea{gen[arOff], d};
 355 }
 356
 357 void debuggerRetImpl(Vout& v, Vreg ar) {
 358   auto const soff = v.makeReg();
 359
 360   v << loadl{ar[AROFF(m_soff)], soff};
 361   v << storel{soff, rvmtl()[unwinderDebuggerReturnOffOff()]};
 362   v << store{rvmsp(), rvmtl()[unwinderDebuggerReturnSPOff()]};
 363
 364   auto const ret = v.makeReg();
 365   v << simplecall(v, unstashDebuggerCatch, ar, ret);
 366
 367   v << jmpr{ret};
 368 }
 369
 370 TCA emitInterpRet(CodeBlock& cb) {
 371   alignJmpTarget(cb);
 372
 373   auto const start = vwrap(cb, [] (Vout& v) {
 374     assertNativeStackAligned(v);
 375     v << lea{rvmsp()[-AROFF(m_r)], r_svcreq_arg(0)};
 376     v << copy{rvmfp(), r_svcreq_arg(1)};
 377   });
 378   svcreq::emit_persistent(cb, folly::none, REQ_POST_INTERP_RET);
 379   return start;
 380 }
 381
 382 template<bool async>
 383 TCA emitInterpGenRet(CodeBlock& cb) {
 384   alignJmpTarget(cb);
 385
 386   auto const start = vwrap(cb, [] (Vout& v) {
 387     assertNativeStackAligned(v);
 388     loadGenFrame<async>(v, r_svcreq_arg(0));
 389     v << copy{rvmfp(), r_svcreq_arg(1)};
 390   });
 391   svcreq::emit_persistent(cb, folly::none, REQ_POST_INTERP_RET);
 392   return start;
 393 }
 394
 395 TCA emitDebuggerInterpRet(CodeBlock& cb) {
 396   alignJmpTarget(cb);
 397
 398   return vwrap(cb, [] (Vout& v) {
 399     assertNativeStackAligned(v);
 400
 401     auto const ar = v.makeReg();
 402     v << lea{rvmsp()[-AROFF(m_r)], ar};
 403     debuggerRetImpl(v, ar);
 404   });
 405 }
 406
 407 template<bool async>
 408 TCA emitDebuggerInterpGenRet(CodeBlock& cb) {
 409   alignJmpTarget(cb);
 410
 411   return vwrap(cb, [] (Vout& v) {
 412     assertNativeStackAligned(v);
 413
 414     auto const ar = v.makeReg();
 415     loadGenFrame<async>(v, ar);
 416     debuggerRetImpl(v, ar);
 417   });
 418 }
 419
 420 ///////////////////////////////////////////////////////////////////////////////
 421
 422 using AFWH = c_AsyncFunctionWaitHandle;
 423
 424 /*
 425  * Convert an AsyncFunctionWaitHandle-relative offset to an offset relative to
 426  * either its contained ActRec or AsioBlockable.
 427  */
 428 constexpr ptrdiff_t ar_rel(ptrdiff_t off) {
 429   return off - AFWH::arOff();
 430 }
 431 constexpr ptrdiff_t bl_rel(ptrdiff_t off) {
 432   return off - AFWH::childrenOff() - AFWH::Node::blockableOff();
 433 }
 434
 435 /*
 436  * Store the async function's return value to the AsyncFunctionWaitHandle.
 437  */
 438 void storeAFWHResult(Vout& v, PhysReg data, PhysReg type) {
 439   auto const resultOff = ar_rel(AFWH::resultOff());
 440   v << store{data, rvmfp()[resultOff + TVOFF(m_data)]};
 441   v << storeb{type, rvmfp()[resultOff + TVOFF(m_type)]};
 442 }
 443
 444 /*
 445  * In a cold path, call into native code to unblock every member of an async
 446  * function's dependency chain, if it has any.
 447  */
 448 void unblockParents(Vout& v, Vout& vcold, Vreg parent) {
 449   auto const sf = v.makeReg();
 450   v << testq{parent, parent, sf};
 451
 452   unlikelyIfThen(v, vcold, CC_NZ, sf, [&] (Vout& v) {
 453     v << vcall{CallSpec::direct(AsioBlockableChain::Unblock),
 454                v.makeVcallArgs({{parent}}), v.makeTuple({})};
 455   });
 456 }
 457
 458 TCA emitAsyncRetCtrl(CodeBlock& cb) {
 459   alignJmpTarget(cb);
 460
 461   return vwrap2(cb, [] (Vout& v, Vout& vcold) {
 462     auto const data = rarg(0);
 463     auto const type = rarg(1);
 464
 465     auto const slow_path = Vlabel(v.makeBlock());
 466
 467     // Load the parent chain.
 468     auto const parentBl = v.makeReg();
 469     v << load{rvmfp()[ar_rel(AFWH::parentChainOff())], parentBl};
 470
 471     // Set state to succeeded.
 472     v << storebi{
 473       c_WaitHandle::toKindState(
 474         c_WaitHandle::Kind::AsyncFunction,
 475         c_WaitHandle::STATE_SUCCEEDED
 476       ),
 477       rvmfp()[ar_rel(c_WaitHandle::stateOff())]
 478     };
 479
 480     // Load the WaitHandle*.
 481     auto const wh = v.makeReg();
 482     v << lea{rvmfp()[Resumable::dataOff() - Resumable::arOff()], wh};
 483
 484     // Check if there's any parent.
 485     auto const hasParent = v.makeReg();
 486     v << testq{parentBl, parentBl, hasParent};
 487     ifThen(v, CC_Z, hasParent, slow_path);
 488
 489     // Check parentBl->getKind() == AFWH.
 490     static_assert(
 491       uint8_t(AsioBlockable::Kind::AsyncFunctionWaitHandleNode) == 0,
 492       "AFWH kind must be 0."
 493     );
 494     auto const isAFWH = v.makeReg();
 495     v << testbim{0x7, parentBl[AsioBlockable::bitsOff()], isAFWH};
 496     ifThen(v, CC_NZ, isAFWH, slow_path);
 497
 498     // Check parentBl->getBWH()->getKindState() == {Async, BLOCKED}.
 499     auto const blockedState = AFWH::toKindState(
 500       c_WaitHandle::Kind::AsyncFunction,
 501       AFWH::STATE_BLOCKED
 502     );
 503     auto const isBlocked = v.makeReg();
 504     v << cmpbim{blockedState, parentBl[bl_rel(AFWH::stateOff())], isBlocked};
 505     ifThen(v, CC_NE, isBlocked, slow_path);
 506
 507     // Check parentBl->getBWH()->resumable()->resumeAddr() != nullptr.
 508     auto const isNullAddr = v.makeReg();
 509     v << cmpqim{0, parentBl[bl_rel(AFWH::resumeAddrOff())], isNullAddr};
 510     ifThen(v, CC_E, isNullAddr, slow_path);
 511
 512     // CheckparentBl->getContextIdx() == child->getContextIdx().
 513     auto const childContextIdx = v.makeReg();
 514     auto const parentContextIdx = v.makeReg();
 515     auto const inSameContext = v.makeReg();
 516
 517     v << loadb{rvmfp()[ar_rel(AFWH::contextIdxOff())], childContextIdx};
 518     v << loadb{parentBl[bl_rel(AFWH::contextIdxOff())], parentContextIdx};
 519     v << cmpb{parentContextIdx, childContextIdx, inSameContext};
 520     ifThen(v, CC_NE, inSameContext, slow_path);
 521
 522     /*
 523      * Fast path.
 524      *
 525      * Handle the return value, unblock any additional parents, release the
 526      * WaitHandle, and transfer control to the parent.
 527      */
 528     // Incref the return value.  In addition to pushing the it onto the stack,
 529     // we are also storing it in the AFWH object.
 530     emitIncRefWork(v, data, type);
 531
 532     // Write the return value to the stack and the AFWH object.
 533     v << storeb{type, rvmsp()[TVOFF(m_type)]};
 534     v << store{data, rvmsp()[TVOFF(m_data)]};
 535     storeAFWHResult(v, data, type);
 536
 537     // Load the next parent in the chain, and unblock the whole chain.
 538     auto const nextParent = v.makeReg();
 539     auto const tmp = v.makeReg();
 540     v << load{parentBl[AsioBlockable::bitsOff()], tmp};
 541     v << andqi{~0x7, tmp, nextParent, v.makeReg()};
 542     unblockParents(v, vcold, nextParent);
 543
 544     // Set up PHP frame linkage for our parent by copying our ActRec's sfp.
 545     auto const sfp = v.makeReg();
 546     v << load{rvmfp()[AROFF(m_sfp)], sfp};
 547     v << store{sfp, parentBl[bl_rel(AFWH::arOff()) + AROFF(m_sfp)]};
 548
 549     // Drop the reference to the current AFWH twice:
 550     //  - it is no longer being executed
 551     //  - it is no longer referenced by the parent
 552     //
 553     // The first time we don't need to check for release.  The second time, we
 554     // do, but we can type-specialize.
 555     emitDecRef(v, wh);
 556     emitDecRefWorkObj(v, wh);
 557
 558     // Update vmfp() and vmFirstAR().
 559     v << lea{parentBl[bl_rel(AFWH::arOff())], rvmfp()};
 560     v << store{rvmfp(), rvmtl()[rds::kVmFirstAROff]};
 561
 562     // setState(STATE_RUNNING)
 563     auto const runningState = c_WaitHandle::toKindState(
 564       c_WaitHandle::Kind::AsyncFunction,
 565       c_ResumableWaitHandle::STATE_RUNNING
 566     );
 567     v << storebi{runningState, parentBl[bl_rel(AFWH::stateOff())]};
 568
 569     // Transfer control to the resume address.
 570     v << jmpm{rvmfp()[ar_rel(AFWH::resumeAddrOff())], php_return_regs()};
 571
 572     /*
 573      * Slow path: unblock all parents, and return to the scheduler.
 574      */
 575     v = slow_path;
 576
 577     // Store result into the AFWH object and unblock all parents.
 578     //
 579     // Storing the result into the AFWH overwrites contextIdx (they share a
 580     // union), so it has to be done after the checks in the fast path (but
 581     // before unblocking parents).
 582     storeAFWHResult(v, data, type);
 583     unblockParents(v, vcold, parentBl);
 584
 585     // Load the saved frame pointer from the ActRec.
 586     v << load{rvmfp()[AROFF(m_sfp)], rvmfp()};
 587
 588     // Decref the WaitHandle.  We only do it once here (unlike in the fast
 589     // path) because the scheduler drops the other reference.
 590     emitDecRefWorkObj(v, wh);
 591
 592     // Adjust stack: on slow path, retVal is not pushed on stack yet.
 593     auto const sync_sp = v.makeReg();
 594     v << lea{rvmsp()[cellsToBytes(1)], sync_sp};
 595     v << syncvmsp{sync_sp};
 596
 597     v << leavetc{php_return_regs()};
 598   });
 599 }
 600
 601 ///////////////////////////////////////////////////////////////////////////////
 602
 603 template<bool immutable>
 604 TCA emitBindCallStub(CodeBlock& cb) {
 605   return vwrap(cb, [] (Vout& v) {
 606     v << phplogue{rvmfp()};
 607
 608     auto args = VregList { v.makeReg(), v.makeReg(),
 609                            v.makeReg(), v.makeReg() };
 610     loadMCG(v, args[0]);
 611
 612     // Reconstruct the address of the call from the saved RIP.
 613     auto const savedRIP = v.makeReg();
 614     auto const callLen = safe_cast<int>(smashableCallLen());
 615     v << load{rvmfp()[AROFF(m_savedRip)], savedRIP};
 616     v << subqi{callLen, savedRIP, args[1], v.makeReg()};
 617
 618     v << copy{rvmfp(), args[2]};
 619     v << movb{v.cns(immutable), args[3]};
 620
 621     auto const handler = reinterpret_cast<void (*)()>(
 622       getMethodPtr(&MCGenerator::handleBindCall)
 623     );
 624     auto const ret = v.makeReg();
 625
 626     v << vcall{
 627       CallSpec::direct(handler),
 628       v.makeVcallArgs({args}),
 629       v.makeTuple({ret}),
 630       Fixup{},
 631       DestType::SSA
 632     };
 633
 634     v << tailcallphp{ret, rvmfp(), php_call_regs()};
 635   });
 636 }
 637
 638 TCA emitFCallArrayHelper(CodeBlock& cb, UniqueStubs& us) {
 639   align(cb, nullptr, Alignment::CacheLine, AlignContext::Dead);
 640
 641   TCA ret = vwrap(cb, [] (Vout& v) {
 642     v << movl{v.cns(0), rarg(2)};
 643   });
 644
 645   us.fcallUnpackHelper = vwrap2(cb, [&] (Vout& v, Vout& vcold) {
 646     // We reach fcallArrayHelper in the same context as a func prologue, so
 647     // this should really be a phplogue{}---but we don't need the return
 648     // address in the ActRec until later, and in the event the callee is
 649     // intercepted, we must save it on the stack because the callee frame will
 650     // already have been popped.  So use a stublogue and "convert" it manually
 651     // later.
 652     v << stublogue{};
 653
 654     storeVMRegs(v);
 655
 656     auto const func = v.makeReg();
 657     auto const unit = v.makeReg();
 658     auto const bc = v.makeReg();
 659
 660     // Load fp->m_func->m_unit->m_bc.
 661     v << load{rvmfp()[AROFF(m_func)], func};
 662     v << load{func[Func::unitOff()], unit};
 663     v << load{unit[Unit::bcOff()], bc};
 664
 665     auto const pc = v.makeReg();
 666     auto const next = v.makeReg();
 667
 668     // Convert offsets into PCs, and sync the PC.
 669     v << addq{bc, rarg(0), pc, v.makeReg()};
 670     v << store{pc, rvmtl()[rds::kVmpcOff]};
 671     v << addq{bc, rarg(1), next, v.makeReg()};
 672
 673     auto const retAddr = v.makeReg();
 674     v << loadstubret{retAddr};
 675
 676     bool (*helper)(PC, int32_t, void*) = &doFCallArrayTC;
 677     v << copyargs{
 678       v.makeTuple({next, rarg(2), retAddr}),
 679       v.makeTuple({rarg(0), rarg(1), rarg(2)})
 680     };
 681     v << call{TCA(helper), arg_regs(3), &us.fcallArrayReturn};
 682     v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 683
 684     auto const sf = v.makeReg();
 685     v << testb{rret(), rret(), sf};
 686
 687     unlikelyIfThen(v, vcold, CC_Z, sf, [&] (Vout& v) {
 688       // If false was returned, we should skip the callee.  The interpreter
 689       // will have popped the pre-live ActRec already, so we can just return to
 690       // the caller.
 691       v << stubret{};
 692     });
 693     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 694
 695     // If true was returned, we're calling the callee, so undo the stublogue{}
 696     // and convert to a phplogue{}. The m_savedRip will be set during the call
 697     // to doFCallArrayTC.
 698     v << stubtophp{};
 699
 700     auto const callee = v.makeReg();
 701     auto const body = v.makeReg();
 702
 703     v << load{rvmfp()[AROFF(m_func)], callee};
 704     emitLdLowPtr(v, callee[Func::funcBodyOff()], body, sizeof(LowPtr<uint8_t>));
 705
 706     // We jmp directly to the func body---this keeps the return stack buffer
 707     // balanced between the call to this stub and the ret from the callee.
 708     v << jmpr{body};
 709   });
 710
 711   return ret;
 712 }
 713
 714 TCA emitFCallArrayEndCatch(CodeBlock& cb, UniqueStubs& us) {
 715   return vwrap(cb, [&] (Vout& v) {
 716     // The CallArray that triggered the call to doFCallArrayTC has a catch trace
 717     // which needs to be run. Switch to a phplogue context to enter the catch.
 718     v << stubtophp{};
 719     loadVMRegs(v);
 720
 721     always_assert(us.endCatchHelper);
 722     v << jmpi{us.endCatchHelper};
 723   });
 724 }
 725
 726 ///////////////////////////////////////////////////////////////////////////////
 727
 728 struct ResumeHelperEntryPoints {
 729   TCA resumeHelperRet;
 730   TCA resumeHelper;
 731   TCA handleResume;
 732   TCA reenterTC;
 733 };
 734
 735 ResumeHelperEntryPoints emitResumeHelpers(CodeBlock& cb) {
 736   ResumeHelperEntryPoints rh;
 737
 738   rh.resumeHelperRet = vwrap(cb, [] (Vout& v) {
 739     v << phplogue{rvmfp()};
 740   });
 741   rh.resumeHelper = vwrap(cb, [] (Vout& v) {
 742     v << ldimmb{0, rarg(1)};
 743   });
 744
 745   rh.handleResume = vwrap(cb, [] (Vout& v) {
 746     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 747     loadMCG(v, rarg(0));
 748
 749     auto const handler = reinterpret_cast<TCA>(
 750       getMethodPtr(&MCGenerator::handleResume)
 751     );
 752     v << call{handler, arg_regs(2)};
 753   });
 754
 755   rh.reenterTC = vwrap(cb, [] (Vout& v) {
 756     loadVMRegs(v);
 757     v << jmpr{rret()};
 758   });
 759
 760   return rh;
 761 }
 762
 763 TCA emitResumeInterpHelpers(CodeBlock& cb, UniqueStubs& us,
 764                             ResumeHelperEntryPoints& rh) {
 765   alignJmpTarget(cb);
 766
 767   rh = emitResumeHelpers(cb);
 768
 769   us.resumeHelperRet = rh.resumeHelperRet;
 770   us.resumeHelper = rh.resumeHelper;
 771
 772   us.interpHelper = vwrap(cb, [] (Vout& v) {
 773     v << store{rarg(0), rvmtl()[rds::kVmpcOff]};
 774   });
 775   us.interpHelperSyncedPC = vwrap(cb, [&] (Vout& v) {
 776     storeVMRegs(v);
 777     v << ldimmb{1, rarg(1)};
 778     v << jmpi{rh.handleResume, RegSet(rarg(1))};
 779   });
 780
 781   us.fcallAwaitSuspendHelper = vwrap(cb, [&] (Vout& v) {
 782     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 783     loadMCG(v, rarg(0));
 784
 785     auto const handler = reinterpret_cast<TCA>(
 786       getMethodPtr(&MCGenerator::handleFCallAwaitSuspend)
 787     );
 788     v << call{handler, arg_regs(2)};
 789     v << jmpi{rh.reenterTC, RegSet()};
 790   });
 791
 792   return us.resumeHelperRet;
 793 }
 794
 795 TCA emitInterpOneCFHelper(CodeBlock& cb, Op op,
 796                           const ResumeHelperEntryPoints& rh) {
 797   alignJmpTarget(cb);
 798
 799   return vwrap(cb, [&] (Vout& v) {
 800     v << copy2{rvmfp(), rvmsp(), rarg(0), rarg(1)};
 801     // rarg(2) is set at the stub callsite.
 802
 803     auto const handler = reinterpret_cast<TCA>(
 804       interpOneEntryPoints[static_cast<size_t>(op)]
 805     );
 806     v << call{handler, arg_regs(3)};
 807
 808     auto const sf = v.makeReg();
 809     auto const next = v.makeBlock();
 810
 811     v << testq{rret(), rret(), sf};
 812     v << jcci{CC_NZ, sf, next, rh.reenterTC};
 813     v = next;
 814     v << jmpi{rh.resumeHelper};
 815   });
 816 }
 817
 818 void emitInterpOneCFHelpers(CodeBlock& cb, UniqueStubs& us,
 819                             const ResumeHelperEntryPoints& rh,
 820                             const CodeCache& code, Debug::DebugInfo& dbg) {
 821   alignJmpTarget(cb);
 822
 823   auto const emit = [&] (Op op, const char* name) {
 824     auto const stub = emitInterpOneCFHelper(cb, op, rh);
 825     us.interpOneCFHelpers[op] = stub;
 826     us.add(name, stub, code, dbg);
 827   };
 828
 829 #define O(name, imm, in, out, flags)          \
 830   if (((flags) & CF) || ((flags) & TF)) {     \
 831     emit(Op::name, "interpOneCFHelper"#name); \
 832   }
 833   OPCODES
 834 #undef O
 835
 836   // Exit is a very special snowflake.  Because it can appear in PHP
 837   // expressions, the emitter pretends that it pushed a value on the eval stack
 838   // (and iopExit actually does push Null right before throwing).  Marking it
 839   // as TF would mess up any bytecodes that want to consume its output value,
 840   // so we can't do that.  But we also don't want to extend regions past it, so
 841   // the JIT treats it as terminal and uses InterpOneCF to execute it.
 842   emit(Op::Exit, "interpOneCFHelperExit");
 843 }
 844
 845 ///////////////////////////////////////////////////////////////////////////////
 846
 847 TCA emitDecRefGeneric(CodeBlock& cb) {
 848   CGMeta meta;
 849
 850   auto const start = vwrap(cb, meta, [] (Vout& v) {
 851     v << stublogue{};
 852
 853     auto const rdata = rarg(0);
 854     auto const rtype = rarg(1);
 855
 856     auto const destroy = [&] (Vout& v) {
 857       // decRefGeneric is called via callfaststub, whose ABI claims that all
 858       // registers are preserved.  This is true in the fast path, but in the
 859       // slow path we need to manually save caller-saved registers.
 860       auto const callerSaved = abi().gpUnreserved - abi().calleeSaved;
 861       PhysRegSaver prs{v, callerSaved};
 862
 863       // As a consequence of being called via callfaststub, we can't safely use
 864       // any Vregs here except for status flags registers, at least not with
 865       // the default vwrap() ABI.  Just use the argument registers instead.
 866       assertx(callerSaved.contains(rdata));
 867       assertx(callerSaved.contains(rtype));
 868
 869       v << movzbq{rtype, rtype};
 870       v << shrli{kShiftDataTypeToDestrIndex, rtype, rtype, v.makeReg()};
 871
 872       auto const dtor_table =
 873         safe_cast<int>(reinterpret_cast<intptr_t>(g_destructors));
 874       v << callm{baseless(rtype * 8 + dtor_table), arg_regs(1)};
 875
 876       // The stub frame's saved RIP is at %rsp[8] before we saved the
 877       // caller-saved registers.
 878       v << syncpoint{makeIndirectFixup(prs.dwordsPushed() + 1)};
 879     };
 880
 881     emitDecRefWork(v, v, rdata, destroy, false);
 882     v << stubret{};
 883   });
 884
 885   meta.process(nullptr);
 886   return start;
 887 }
 888
 889 ///////////////////////////////////////////////////////////////////////////////
 890
 891 TCA emitHandleSRHelper(CodeBlock& cb) {
 892   alignJmpTarget(cb);
 893
 894   return vwrap(cb, [] (Vout& v) {
 895     storeVMRegs(v);
 896
 897     // Pack the service request args into a svcreq::ReqInfo on the stack.
 898     for (auto i = svcreq::kMaxArgs; i-- > 0; ) {
 899       v << push{r_svcreq_arg(i)};
 900     }
 901     v << push{r_svcreq_stub()};
 902     v << push{r_svcreq_req()};
 903
 904     // Call mcg->handleServiceRequest(rsp()).
 905     auto const args = VregList { v.makeReg(), v.makeReg() };
 906     loadMCG(v, args[0]);
 907     v << copy{rsp(), args[1]};
 908
 909     auto const meth = &MCGenerator::handleServiceRequest;
 910     auto const ret = v.makeReg();
 911
 912     v << vcall{
 913       CallSpec::method(meth),
 914       v.makeVcallArgs({args}),
 915       v.makeTuple({ret}),
 916       Fixup{},
 917       DestType::SSA
 918     };
 919
 920     // Pop the ReqInfo off the stack.
 921     auto const reqinfo_sz = static_cast<int>(sizeof(svcreq::ReqInfo));
 922     v << lea{rsp()[reqinfo_sz], rsp()};
 923
 924     // rvmtl() was preserved by the callee, but rvmsp() and rvmfp() might've
 925     // changed if we interpreted anything.  Reload them.
 926     loadVMRegs(v);
 927
 928     v << jmpr{ret};
 929   });
 930 }
 931
 932 TCA emitThrowSwitchMode(CodeBlock& cb) {
 933   alignJmpTarget(cb);
 934
 935   return vwrap(cb, [] (Vout& v) {
 936     v << call{TCA(throwSwitchMode)};
 937     v << ud2{};
 938   });
 939 }
 940
 941 ///////////////////////////////////////////////////////////////////////////////
 942
 943 }
 944
 945 ///////////////////////////////////////////////////////////////////////////////
 946
 947 void UniqueStubs::emitAll(CodeCache& code, Debug::DebugInfo& dbg) {
 948   auto view = code.view();
 949   auto& main = view.main();
 950   auto& cold = view.cold();
 951   auto& frozen = view.frozen();
 952   auto& hotBlock = code.view(true /* hot */).main();
 953
 954   auto const hot = [&]() -> CodeBlock& {
 955     return hotBlock.available() > 512 ? hotBlock : main;
 956   };
 957
 958 #define ADD(name, stub) name = add(#name, (stub), code, dbg)
 959   ADD(handleSRHelper, emitHandleSRHelper(cold)); // required by emitInterpRet()
 960
 961   ADD(funcPrologueRedispatch, emitFuncPrologueRedispatch(hot()));
 962   ADD(fcallHelperThunk,       emitFCallHelperThunk(cold));
 963   ADD(funcBodyHelperThunk,    emitFuncBodyHelperThunk(cold));
 964   ADD(functionEnterHelper,    emitFunctionEnterHelper(cold, *this));
 965   ADD(functionSurprisedOrStackOverflow,
 966       emitFunctionSurprisedOrStackOverflow(cold, *this));
 967
 968   ADD(retHelper,                  emitInterpRet(cold));
 969   ADD(genRetHelper,               emitInterpGenRet<false>(cold));
 970   ADD(asyncGenRetHelper,          emitInterpGenRet<true>(cold));
 971   ADD(retInlHelper,               emitInterpRet(cold));
 972   ADD(asyncRetCtrl,               emitAsyncRetCtrl(main));
 973   ADD(debuggerRetHelper,          emitDebuggerInterpRet(cold));
 974   ADD(debuggerGenRetHelper,       emitDebuggerInterpGenRet<false>(cold));
 975   ADD(debuggerAsyncGenRetHelper,  emitDebuggerInterpGenRet<true>(cold));
 976
 977   ADD(bindCallStub,           emitBindCallStub<false>(cold));
 978   ADD(immutableBindCallStub,  emitBindCallStub<true>(cold));
 979   ADD(fcallArrayHelper,       emitFCallArrayHelper(hot(), *this));
 980
 981   ADD(decRefGeneric,  emitDecRefGeneric(cold));
 982
 983   ADD(callToExit,       emitCallToExit(main));
 984   ADD(endCatchHelper,   emitEndCatchHelper(frozen, *this));
 985   ADD(throwSwitchMode,  emitThrowSwitchMode(frozen));
 986
 987   ADD(fcallArrayEndCatch, emitFCallArrayEndCatch(frozen, *this));
 988 #undef ADD
 989
 990   add("freeLocalsHelpers", emitFreeLocalsHelpers(hot(), *this), code, dbg);
 991
 992   ResumeHelperEntryPoints rh;
 993   add("resumeInterpHelpers",
 994       emitResumeInterpHelpers(main, *this, rh),
 995       code, dbg);
 996   emitInterpOneCFHelpers(cold, *this, rh, code, dbg);
 997 }
 998
 999 ///////////////////////////////////////////////////////////////////////////////
1000
1001 TCA UniqueStubs::add(const char* name, TCA start,
1002                      const CodeCache& code, Debug::DebugInfo& dbg) {
1003   auto& cb = code.blockFor(start);
1004   auto const end = cb.frontier();
1005
1006   FTRACE(1, "unique stub: {} @ {} -- {:4} bytes: {}\n",
1007          cb.name(),
1008          static_cast<void*>(start),
1009          static_cast<size_t>(end - start),
1010          name);
1011
1012   ONTRACE(2,
1013           [&]{
1014             Disasm dasm(Disasm::Options().indent(4));
1015             std::ostringstream os;
1016             dasm.disasm(os, start, end);
1017             FTRACE(2, "{}\n", os.str());
1018           }()
1019          );
1020
1021   if (!RuntimeOption::EvalJitNoGdb) {
1022     dbg.recordStub(Debug::TCRange(start, end, &cb == &code.cold()),
1023                    folly::sformat("HHVM::{}", name));
1024   }
1025
1026   auto const newStub = StubRange{name, start, end};
1027   auto lower = std::lower_bound(m_ranges.begin(), m_ranges.end(), newStub);
1028
1029   // We assume ranges are non-overlapping.
1030   assertx(lower == m_ranges.end() || newStub.end <= lower->start);
1031   assertx(lower == m_ranges.begin() || (lower - 1)->end <= newStub.start);
1032   m_ranges.insert(lower, newStub);
1033   return start;
1034 }
1035
1036 std::string UniqueStubs::describe(TCA address) const {
1037   auto raw = [address] { return folly::sformat("{}", address); };
1038   if (m_ranges.empty()) return raw();
1039
1040   auto const dummy = StubRange{"", address, nullptr};
1041   auto lower = std::upper_bound(m_ranges.begin(), m_ranges.end(), dummy);
1042   if (lower == m_ranges.begin()) return raw();
1043
1044   --lower;
1045   if (lower->contains(address)) {
1046     return folly::sformat("{}+{:#x}", lower->name, address - lower->start);
1047   }
1048   return raw();
1049 }
1050
1051 ///////////////////////////////////////////////////////////////////////////////
1052
1053 RegSet interp_one_cf_regs() {
1054   return vm_regs_with_sp() | rarg(2);
1055 }
1056
1057 void emitInterpReq(Vout& v, SrcKey sk, FPInvOffset spOff) {
1058   if (RuntimeOption::EvalJitTransCounters) emitTransCounterInc(v);
1059
1060   if (!sk.resumed()) {
1061     v << lea{rvmfp()[-cellsToBytes(spOff.offset)], rvmsp()};
1062   }
1063   v << copy{v.cns(sk.pc()), rarg(0)};
1064   v << jmpi{mcg->ustubs().interpHelper, arg_regs(1)};
1065 }
1066
1067 ///////////////////////////////////////////////////////////////////////////////
1068
1069 void enterTCImpl(TCA start, ActRec* stashedAR) {
1070   ARCH_SWITCH_CALL(enterTCImpl, start, stashedAR);
1071 }
1072
1073 ///////////////////////////////////////////////////////////////////////////////
1074
1075 }}