hphp/runtime/vm/jit/unique-stubs.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/unique-stubs.h"
  18
  19 #include "hphp/runtime/base/datatype.h"
  20 #include "hphp/runtime/base/rds-header.h"
  21 #include "hphp/runtime/base/runtime-option.h"
  22 #include "hphp/runtime/base/surprise-flags.h"
  23 #include "hphp/runtime/base/tv-mutate.h"
  24 #include "hphp/runtime/base/tv-variant.h"
  25 #include "hphp/runtime/vm/bytecode.h"
  26 #include "hphp/runtime/vm/debug/debug.h"
  27 #include "hphp/runtime/vm/event-hook.h"
  28 #include "hphp/runtime/vm/hhbc.h"
  29 #include "hphp/runtime/vm/interp-helpers.h"
  30 #include "hphp/runtime/vm/srckey.h"
  31 #include "hphp/runtime/vm/vm-regs.h"
  32
  33 #include "hphp/runtime/vm/jit/types.h"
  34 #include "hphp/runtime/vm/jit/abi.h"
  35 #include "hphp/runtime/vm/jit/align.h"
  36 #include "hphp/runtime/vm/jit/cg-meta.h"
  37 #include "hphp/runtime/vm/jit/code-cache.h"
  38 #include "hphp/runtime/vm/jit/code-gen-cf.h"
  39 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  40 #include "hphp/runtime/vm/jit/code-gen-tls.h"
  41 #include "hphp/runtime/vm/jit/debugger.h"
  42 #include "hphp/runtime/vm/jit/fixup.h"
  43 #include "hphp/runtime/vm/jit/mcgen.h"
  44 #include "hphp/runtime/vm/jit/phys-reg.h"
  45 #include "hphp/runtime/vm/jit/phys-reg-saver.h"
  46 #include "hphp/runtime/vm/jit/print.h"
  47 #include "hphp/runtime/vm/jit/service-request-handlers.h"
  48 #include "hphp/runtime/vm/jit/service-requests.h"
  49 #include "hphp/runtime/vm/jit/smashable-instr.h"
  50 #include "hphp/runtime/vm/jit/stack-offsets.h"
  51 #include "hphp/runtime/vm/jit/stack-overflow.h"
  52 #include "hphp/runtime/vm/jit/target-cache.h"
  53 #include "hphp/runtime/vm/jit/tc.h"
  54 #include "hphp/runtime/vm/jit/translator-inline.h"
  55 #include "hphp/runtime/vm/jit/unique-stubs-arm.h"
  56 #include "hphp/runtime/vm/jit/unique-stubs-ppc64.h"
  57 #include "hphp/runtime/vm/jit/unique-stubs-x64.h"
  58 #include "hphp/runtime/vm/jit/unwind-itanium.h"
  59 #include "hphp/runtime/vm/jit/vasm-gen.h"
  60 #include "hphp/runtime/vm/jit/vasm-instr.h"
  61 #include "hphp/runtime/vm/jit/vasm-reg.h"
  62 #include "hphp/runtime/vm/jit/vtune-jit.h"
  63
  64 #include "hphp/runtime/ext/asio/ext_async-generator.h"
  65 #include "hphp/runtime/ext/generator/ext_generator.h"
  66
  67 #include "hphp/util/abi-cxx.h"
  68 #include "hphp/util/arch.h"
  69 #include "hphp/util/asm-x64.h"
  70 #include "hphp/util/data-block.h"
  71 #include "hphp/util/trace.h"
  72
  73 #include <folly/Format.h>
  74
  75 #include <algorithm>
  76
  77 namespace HPHP { namespace jit {
  78
  79 ///////////////////////////////////////////////////////////////////////////////
  80
  81 TRACE_SET_MOD(ustubs);
  82
  83 ///////////////////////////////////////////////////////////////////////////////
  84
  85 namespace {
  86
  87 ///////////////////////////////////////////////////////////////////////////////
  88
  89 void alignJmpTarget(CodeBlock& cb) {
  90   if (RuntimeOption::EvalJitAlignUniqueStubs) {
  91     align(cb, nullptr, Alignment::JmpTarget, AlignContext::Dead);
  92   }
  93 }
  94
  95 void alignCacheLine(CodeBlock& cb) {
  96   if (RuntimeOption::EvalJitAlignUniqueStubs) {
  97     align(cb, nullptr, Alignment::CacheLine, AlignContext::Dead);
  98   }
  99 }
 100
 101 void assertNativeStackAligned(Vout& v) {
 102   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 103     v << call{TCA(assert_native_stack_aligned)};
 104   }
 105 }
 106
 107 /*
 108  * Load and store the VM registers from/to RDS.
 109  */
 110 void loadVMRegs(Vout& v) {
 111   v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 112   v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 113 }
 114 void storeVMRegs(Vout& v) {
 115   v << store{rvmfp(), rvmtl()[rds::kVmfpOff]};
 116   v << store{rvmsp(), rvmtl()[rds::kVmspOff]};
 117 }
 118
 119 /*
 120  * Load and store the PHP return registers from/to the top of the VM stack.
 121  *
 122  * Note that we don't do loadb{}/storeb{} for the type register, because we
 123  * sometimes need to preserve the m_aux field across returns.
 124  */
 125 void loadReturnRegs(Vout& v) {
 126   v << load{rvmsp()[TVOFF(m_data)], rret_data()};
 127   v << load{rvmsp()[TVOFF(m_type)], rret_type()};
 128 }
 129 void storeReturnRegs(Vout& v) {
 130   v << store{rret_data(), rvmsp()[TVOFF(m_data)]};
 131   v << store{rret_type(), rvmsp()[TVOFF(m_type)]};
 132 }
 133
 134 /*
 135  * Convenience wrapper around a simple vcall to `helper', with a single `arg'
 136  * and a return value in `d'.
 137  */
 138 template<class F>
 139 Vinstr simplecall(Vout& v, F helper, Vreg arg, Vreg d) {
 140   return vcall{
 141     CallSpec::direct(helper),
 142     v.makeVcallArgs({{arg}}),
 143     v.makeTuple({d}),
 144     Fixup{},
 145     DestType::SSA
 146   };
 147 }
 148
 149 /*
 150  * Convenience wrapper around a simple vcall to `helper', with a single `arg'
 151  * and a pair of return values in `d1' and `d2'.
 152  */
 153 template<class F>
 154 Vinstr simplecall(Vout& v, F helper, Vreg arg, Vreg d1, Vreg d2) {
 155   return vcall{
 156     CallSpec::direct(helper),
 157     v.makeVcallArgs({{arg}}),
 158     v.makeTuple({d1, d2}),
 159     Fixup{},
 160     DestType::SSAPair
 161   };
 162 }
 163
 164 /*
 165  * Emit a catch trace that unwinds a stub context back to the PHP context that
 166  * called it.
 167  */
 168 template<class GenFn>
 169 void emitStubCatch(Vout& v, const UniqueStubs& us, GenFn gen) {
 170   always_assert(us.endCatchHelper);
 171   v << landingpad{};
 172   gen(v);
 173   v << stubunwind{};
 174   v << jmpi{us.endCatchHelper};
 175 }
 176
 177 ///////////////////////////////////////////////////////////////////////////////
 178
 179 TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 180   alignCacheLine(cb);
 181   return ARCH_SWITCH_CALL(emitFreeLocalsHelpers, cb, data, us);
 182 }
 183
 184 TCA emitCallToExit(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 185   alignJmpTarget(cb);
 186   return ARCH_SWITCH_CALL(emitCallToExit, cb, data, us);
 187 }
 188
 189 ///////////////////////////////////////////////////////////////////////////////
 190
 191 struct FCallHelperRet {
 192   TCA destAddr;
 193   TCA retAddr;
 194 };
 195
 196 FCallHelperRet fcallHelper(ActRec* ar) {
 197   assert_native_stack_aligned();
 198   assertx(!ar->resumed());
 199
 200   if (LIKELY(!RuntimeOption::EvalFailJitPrologs)) {
 201     auto const tca = mcgen::getFuncPrologue(
 202       const_cast<Func*>(ar->func()),
 203       ar->numArgs()
 204     );
 205     if (tca) return { tca, nullptr };
 206   }
 207
 208   // Check for stack overflow in the same place func prologues make their
 209   // StackCheck::Early check (see irgen-func-prologue.cpp).  This handler also
 210   // cleans and syncs vmRegs for us.
 211   if (checkCalleeStackOverflow(ar)) handleStackOverflow(ar);
 212
 213   // If doFCall indicates that the function was intercepted and should be
 214   // skipped, it will have already torn down the callee's frame. So, we need to
 215   // save the return value thats in it.
 216   auto const retAddr = (TCA)ar->m_savedRip;
 217
 218   try {
 219     VMRegAnchor _(ar);
 220     if (doFCall(ar, vmpc())) {
 221       return { tc::ustubs().resumeHelperRet, nullptr };
 222     }
 223     // We've been asked to skip the function body (fb_intercept).  The vmregs
 224     // have already been fixed; indicate this with a nullptr return.
 225     return { nullptr, retAddr };
 226   } catch (...) {
 227     // The VMRegAnchor above took care of us, but we need to tell the unwinder
 228     // (since ~VMRegAnchor() will have reset tl_regState).
 229     tl_regState = VMRegState::CLEAN;
 230     throw;
 231   }
 232 }
 233
 234 ///////////////////////////////////////////////////////////////////////////////
 235
 236 TCA emitFuncPrologueRedispatch(CodeBlock& cb, DataBlock& data) {
 237   alignCacheLine(cb);
 238
 239   return vwrap(cb, data, [] (Vout& v) {
 240     auto const func = v.makeReg();
 241     v << load{rvmfp()[AROFF(m_func)], func};
 242
 243     auto const argc = v.makeReg();
 244     auto const naaf = v.makeReg();
 245     v << loadl{rvmfp()[AROFF(m_numArgsAndFlags)], naaf};
 246     v << andli{ActRec::kNumArgsMask, naaf, argc, v.makeReg()};
 247
 248     auto const nparams = v.makeReg();
 249     auto const pcounts = v.makeReg();
 250     v << loadl{func[Func::paramCountsOff()], pcounts};
 251     v << shrli{0x1, pcounts, nparams, v.makeReg()};
 252
 253     auto const sf = v.makeReg();
 254     v << cmpl{argc, nparams, sf};
 255
 256     auto const pTabOff = safe_cast<int32_t>(Func::prologueTableOff());
 257     auto const ptrSize = safe_cast<int32_t>(sizeof(LowPtr<uint8_t>));
 258
 259     // If we passed more args than declared, we need to dispatch to the
 260     // "too many arguments" prologue.
 261     ifThen(v, CC_L, sf, [&] (Vout& v) {
 262       auto const dest = v.makeReg();
 263
 264       auto const nargs = v.makeReg();
 265       v << movzlq{nparams, nargs};
 266
 267       emitLdLowPtr(v, func[nargs * ptrSize + (pTabOff + ptrSize)],
 268                    dest, sizeof(LowPtr<uint8_t>));
 269       v << jmpr{dest};
 270     });
 271
 272     auto const nargs = v.makeReg();
 273     v << movzlq{argc, nargs};
 274
 275     auto const dest = v.makeReg();
 276     emitLdLowPtr(v, func[nargs * ptrSize + pTabOff],
 277                  dest, sizeof(LowPtr<uint8_t>));
 278     v << jmpr{dest};
 279   });
 280 }
 281
 282 TCA emitFCallHelperThunk(CodeBlock& main, CodeBlock& cold, DataBlock& data) {
 283   alignJmpTarget(main);
 284
 285   return vwrap2(main, cold, data, [] (Vout& v, Vout& vc) {
 286     v << phplogue{rvmfp()};
 287
 288     // fcallHelper asserts native stack alignment for us.
 289     FCallHelperRet (*helper)(ActRec*) = &fcallHelper;
 290     auto const dest = v.makeReg();
 291     auto const saved_rip = v.makeReg();
 292     v << simplecall(v, helper, rvmfp(), dest, saved_rip);
 293
 294     // Clobber rvmsp in debug builds.
 295     if (debug) v << copy{v.cns(0x1), rvmsp()};
 296
 297     auto const sf = v.makeReg();
 298     v << testq{dest, dest, sf};
 299
 300     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 301       // A nullptr dest means the callee was intercepted and should be
 302       // skipped. In that case, saved_rip will contain the return address that
 303       // was in the callee's ActRec before it was torn down by the intercept.
 304       loadVMRegs(v);
 305       loadReturnRegs(v);
 306
 307       // Return to the caller. This unbalances the return stack buffer, but if
 308       // we're intercepting, we probably don't care.
 309       v << jmpr{saved_rip};
 310     });
 311
 312     // Jump to the func prologue.
 313     v << tailcallphp{dest, rvmfp(), php_call_regs()};
 314   });
 315 }
 316
 317 TCA emitFuncBodyHelperThunk(CodeBlock& cb, DataBlock& data) {
 318   alignJmpTarget(cb);
 319
 320   return vwrap(cb, data, [] (Vout& v) {
 321     TCA (*helper)(ActRec*) = &svcreq::funcBodyHelper;
 322     auto const dest = v.makeReg();
 323     v << simplecall(v, helper, rvmfp(), dest);
 324     v << jmpr{dest};
 325   });
 326 }
 327
 328 TCA emitFunctionEnterHelper(CodeBlock& main, CodeBlock& cold,
 329                             DataBlock& data, UniqueStubs& us) {
 330   alignCacheLine(main);
 331
 332   CGMeta meta;
 333
 334   auto const start = vwrap2(main, cold, data, meta, [&] (Vout& v, Vout& vc) {
 335     auto const ar = v.makeReg();
 336
 337     v << copy{rvmfp(), ar};
 338
 339     // Fully set up the call frame for the stub.  We can't skip this like we do
 340     // in other stubs because we need the return IP for this frame in the %rbp
 341     // chain, in order to find the proper fixup for the VMRegAnchor in the
 342     // intercept handler.
 343     v << stublogue{true};
 344     v << copy{rsp(), rvmfp()};
 345
 346     // When we call the event hook, it might tell us to skip the callee
 347     // (because of fb_intercept).  If that happens, we need to return to the
 348     // caller, but the handler will have already popped the callee's frame.
 349     // So, we need to save these values for later.
 350     v << pushpm{ar[AROFF(m_savedRip)], ar[AROFF(m_sfp)]};
 351
 352     v << copy2{ar, v.cns(EventHook::NormalFunc), rarg(0), rarg(1)};
 353
 354     auto const done = v.makeBlock();
 355     auto const ctch = vc.makeBlock();
 356     auto const should_continue = v.makeReg();
 357     bool (*hook)(const ActRec*, int) = &EventHook::onFunctionCall;
 358
 359     v << vinvoke{
 360       CallSpec::direct(hook),
 361       v.makeVcallArgs({{ar, v.cns(EventHook::NormalFunc)}}),
 362       v.makeTuple({should_continue}),
 363       {done, ctch},
 364       Fixup{},
 365       DestType::SSA
 366     };
 367
 368     vc = ctch;
 369     emitStubCatch(vc, us, [] (Vout& v) {
 370       // Skip past the stuff we saved for the intercept case.
 371       v << lea{rsp()[16], rsp()};
 372       // Undo our stub frame, so that rvmfp() points to the parent VM frame.
 373       v << load{rsp()[AROFF(m_sfp)], rvmfp()};
 374     });
 375
 376     v = done;
 377
 378     auto const sf = v.makeReg();
 379     v << testb{should_continue, should_continue, sf};
 380
 381     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 382       auto const saved_rip = v.makeReg();
 383
 384       // The event hook has already cleaned up the stack and popped the
 385       // callee's frame, so we're ready to continue from the original call
 386       // site.  We just need to grab the fp/rip of the original frame that we
 387       // saved earlier, and sync rvmsp().
 388       v << popp{rvmfp(), saved_rip};
 389
 390       // Drop our call frame; the stublogue{} instruction guarantees that this
 391       // is exactly 16 bytes.
 392       v << lea{rsp()[kNativeFrameSize], rsp()};
 393
 394       // Sync vmsp and the return regs.
 395       v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 396       v << load{rvmsp()[TVOFF(m_data)], rret_data()};
 397       v << load{rvmsp()[TVOFF(m_type)], rret_type()};
 398
 399       // Return to the caller.  This unbalances the return stack buffer, but if
 400       // we're intercepting, we probably don't care.
 401       v << jmpr{saved_rip};
 402     });
 403
 404     // Skip past the stuff we saved for the intercept case.
 405     v << lea{rsp()[16], rsp()};
 406
 407     // Restore rvmfp() and return to the callee's func prologue.
 408     v << stubret{RegSet(), true};
 409   });
 410
 411   meta.process(nullptr);
 412   return start;
 413 }
 414
 415 TCA emitFunctionSurprisedOrStackOverflow(CodeBlock& main,
 416                                          CodeBlock& cold,
 417                                          DataBlock& data,
 418                                          const UniqueStubs& us) {
 419   alignJmpTarget(main);
 420
 421   CGMeta meta;
 422
 423   auto const start = vwrap2(main, cold, data, meta, [&] (Vout& v, Vout& vc) {
 424     v << stublogue{};
 425
 426     auto const done = v.makeBlock();
 427     auto const ctch = vc.makeBlock();
 428
 429     v << vinvoke{CallSpec::direct(handlePossibleStackOverflow),
 430                  v.makeVcallArgs({{rvmfp()}}), v.makeTuple({}),
 431                  {done, ctch}};
 432     vc = ctch;
 433     emitStubCatch(vc, us, [](Vout& /*v*/) {});
 434
 435     v = done;
 436     v << tailcallstub{us.functionEnterHelper};
 437   });
 438
 439   meta.process(nullptr);
 440   return start;
 441 }
 442
 443 ///////////////////////////////////////////////////////////////////////////////
 444
 445 template<bool async>
 446 void loadGenFrame(Vout& v, Vreg d) {
 447   auto const arOff = BaseGenerator::arOff() -
 448     (async ? AsyncGenerator::objectOff() : Generator::objectOff());
 449
 450   auto const gen = v.makeReg();
 451
 452   // We have to get the Generator object from the current frame's $this, then
 453   // load the embedded frame.
 454   v << load{rvmfp()[AROFF(m_thisUnsafe)], gen};
 455   v << lea{gen[arOff], d};
 456 }
 457
 458 void debuggerRetImpl(Vout& v, Vreg ar) {
 459   auto const soff = v.makeReg();
 460
 461   v << loadl{ar[AROFF(m_soff)], soff};
 462   v << storel{soff, rvmtl()[unwinderDebuggerReturnOffOff()]};
 463   v << store{rvmsp(), rvmtl()[unwinderDebuggerReturnSPOff()]};
 464
 465   auto const ret = v.makeReg();
 466   v << simplecall(v, unstashDebuggerCatch, ar, ret);
 467
 468   v << jmpr{ret};
 469 }
 470
 471 TCA emitInterpRet(CodeBlock& cb, DataBlock& data) {
 472   alignCacheLine(cb);
 473
 474   auto const start = vwrap(cb, data, [] (Vout& v) {
 475     // Sync return regs before calling native assert function.
 476     storeReturnRegs(v);
 477     assertNativeStackAligned(v);
 478
 479     v << lea{rvmsp()[-kArRetOff], r_svcreq_arg(0)};
 480     v << copy{rvmfp(), r_svcreq_arg(1)};
 481   });
 482   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_INTERP_RET);
 483   return start;
 484 }
 485
 486 template<bool async>
 487 TCA emitInterpGenRet(CodeBlock& cb, DataBlock& data) {
 488   alignJmpTarget(cb);
 489
 490   auto const start = vwrap(cb, data, [] (Vout& v) {
 491     // Sync return regs before calling native assert function.
 492     storeReturnRegs(v);
 493     assertNativeStackAligned(v);
 494
 495     loadGenFrame<async>(v, r_svcreq_arg(0));
 496     v << copy{rvmfp(), r_svcreq_arg(1)};
 497   });
 498   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_INTERP_RET);
 499   return start;
 500 }
 501
 502 TCA emitDebuggerInterpRet(CodeBlock& cb, DataBlock& data) {
 503   alignJmpTarget(cb);
 504
 505   return vwrap(cb, data, [] (Vout& v) {
 506     // Sync return regs before calling native assert function.
 507     storeReturnRegs(v);
 508     assertNativeStackAligned(v);
 509
 510     auto const ar = v.makeReg();
 511     v << lea{rvmsp()[-kArRetOff], ar};
 512     debuggerRetImpl(v, ar);
 513   });
 514 }
 515
 516 template<bool async>
 517 TCA emitDebuggerInterpGenRet(CodeBlock& cb, DataBlock& data) {
 518   alignJmpTarget(cb);
 519
 520   return vwrap(cb, data, [] (Vout& v) {
 521     assertNativeStackAligned(v);
 522
 523     auto const ar = v.makeReg();
 524     loadGenFrame<async>(v, ar);
 525     debuggerRetImpl(v, ar);
 526   });
 527 }
 528
 529 ///////////////////////////////////////////////////////////////////////////////
 530
 531 template<bool immutable>
 532 TCA emitBindCallStub(CodeBlock& cb, DataBlock& data) {
 533   return vwrap(cb, data, [] (Vout& v) {
 534     v << phplogue{rvmfp()};
 535
 536     auto args = VregList { v.makeReg(), v.makeReg(), v.makeReg() };
 537
 538     // Reconstruct the address of the call from the saved RIP.
 539     auto const savedRIP = v.makeReg();
 540     auto const callLen = safe_cast<int>(smashableCallLen());
 541     v << load{rvmfp()[AROFF(m_savedRip)], savedRIP};
 542     v << subqi{callLen, savedRIP, args[0], v.makeReg()};
 543
 544     v << copy{rvmfp(), args[1]};
 545     v << movb{v.cns(immutable), args[2]};
 546
 547     auto const ret = v.makeReg();
 548
 549     v << vcall{
 550       CallSpec::direct(svcreq::handleBindCall),
 551       v.makeVcallArgs({args}),
 552       v.makeTuple({ret}),
 553       Fixup{},
 554       DestType::SSA
 555     };
 556
 557     v << tailcallphp{ret, rvmfp(), php_call_regs()};
 558   });
 559 }
 560
 561 TCA emitFCallUnpackHelper(CodeBlock& main, CodeBlock& cold,
 562                           DataBlock& data, UniqueStubs& us) {
 563   alignCacheLine(main);
 564
 565   CGMeta meta;
 566
 567   auto const ret = vwrap2(main, cold, data, meta, [&] (Vout& v, Vout& vc) {
 568     // We reach fcallUnpackHelper in the same context as a func prologue, so
 569     // this should really be a phplogue{}---but we don't need the return
 570     // address in the ActRec until later, and in the event the callee is
 571     // intercepted, we must save it on the stack because the callee frame will
 572     // already have been popped.  So use a stublogue and "convert" it manually
 573     // later.
 574     v << stublogue{};
 575
 576     storeVMRegs(v);
 577
 578     auto const func = v.makeReg();
 579     auto const unit = v.makeReg();
 580     auto const bc = v.makeReg();
 581
 582     // Load fp->m_func->m_unit->m_bc.
 583     v << load{rvmfp()[AROFF(m_func)], func};
 584     v << load{func[Func::unitOff()], unit};
 585     v << load{unit[Unit::bcOff()], bc};
 586
 587     auto const pc = v.makeReg();
 588     auto const next = v.makeReg();
 589
 590     // Convert offsets into PCs, and sync the PC.
 591     v << addq{bc, rarg(0), pc, v.makeReg()};
 592     v << store{pc, rvmtl()[rds::kVmpcOff]};
 593     v << addq{bc, rarg(1), next, v.makeReg()};
 594
 595     auto const retAddr = v.makeReg();
 596     v << loadstubret{retAddr};
 597
 598     auto const done = v.makeBlock();
 599     auto const ctch = vc.makeBlock();
 600     auto const should_continue = v.makeReg();
 601     bool (*helper)(PC, int32_t, void*) = &doFCallUnpackTC;
 602
 603     v << vinvoke{
 604       CallSpec::direct(helper),
 605       v.makeVcallArgs({{next, rarg(2), retAddr}}),
 606       v.makeTuple({should_continue}),
 607       {done, ctch},
 608       Fixup{},
 609       DestType::SSA
 610     };
 611     vc = ctch;
 612     emitStubCatch(vc, us, [] (Vout& v) { loadVMRegs(v); });
 613
 614     v = done;
 615
 616     // Load only rvmsp(); we need to wait to make sure we aren't skipping the
 617     // callee before loading rvmfp().
 618     v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 619
 620     auto const sf = v.makeReg();
 621     v << testb{should_continue, should_continue, sf};
 622
 623     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 624       // If false was returned, we should skip the callee.  The interpreter
 625       // will have popped the pre-live ActRec already, so we can just return to
 626       // the caller after syncing the return regs.
 627       loadReturnRegs(v);
 628       v << stubret{};
 629     });
 630     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 631
 632     // If true was returned, we're calling the callee, so undo the stublogue{}
 633     // and convert to a phplogue{}.
 634     v << stubtophp{rvmfp()};
 635
 636     auto const callee = v.makeReg();
 637     auto const body = v.makeReg();
 638
 639     v << load{rvmfp()[AROFF(m_func)], callee};
 640     emitLdLowPtr(v, callee[Func::funcBodyOff()], body, sizeof(LowPtr<uint8_t>));
 641
 642     // We jmp directly to the func body---this keeps the return stack buffer
 643     // balanced between the call to this stub and the ret from the callee.
 644     v << jmpr{body};
 645   });
 646
 647   meta.process(nullptr);
 648   return ret;
 649 }
 650
 651 ///////////////////////////////////////////////////////////////////////////////
 652
 653 struct ResumeHelperEntryPoints {
 654   TCA resumeHelperRet;
 655   TCA resumeHelper;
 656   TCA handleResume;
 657   TCA reenterTC;
 658 };
 659
 660 ResumeHelperEntryPoints emitResumeHelpers(CodeBlock& cb, DataBlock& data) {
 661   ResumeHelperEntryPoints rh;
 662
 663   rh.resumeHelperRet = vwrap(cb, data, [] (Vout& v) {
 664     v << phplogue{rvmfp()};
 665   });
 666   rh.resumeHelper = vwrap(cb, data, [] (Vout& v) {
 667     v << ldimmb{0, rarg(0)};
 668   });
 669
 670   rh.handleResume = vwrap(cb, data, [] (Vout& v) {
 671     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 672
 673     auto const handler = reinterpret_cast<TCA>(svcreq::handleResume);
 674     v << call{handler, arg_regs(2)};
 675   });
 676
 677   rh.reenterTC = vwrap(cb, data, [] (Vout& v) {
 678     // Save the return of handleResume(), then sync regs.
 679     auto const target = v.makeReg();
 680     v << copy{rret(), target};
 681
 682     loadVMRegs(v);
 683     loadReturnRegs(v);  // spurious load if we're not returning
 684
 685     v << jmpr{target};
 686   });
 687
 688   return rh;
 689 }
 690
 691 TCA emitResumeInterpHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us,
 692                             ResumeHelperEntryPoints& rh) {
 693   alignCacheLine(cb);
 694
 695   rh = emitResumeHelpers(cb, data);
 696
 697   us.resumeHelperRet = rh.resumeHelperRet;
 698   us.resumeHelper = rh.resumeHelper;
 699
 700   us.interpHelper = vwrap(cb, data, [] (Vout& v) {
 701     v << store{rarg(0), rvmtl()[rds::kVmpcOff]};
 702   });
 703   us.interpHelperSyncedPC = vwrap(cb, data, [&] (Vout& v) {
 704     storeVMRegs(v);
 705     v << ldimmb{1, rarg(0)};
 706     v << jmpi{rh.handleResume, RegSet(rarg(0))};
 707   });
 708
 709   us.fcallAwaitSuspendHelper = vwrap(cb, data, [&] (Vout& v) {
 710     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 711
 712     auto const handler = reinterpret_cast<TCA>(svcreq::handleFCallAwaitSuspend);
 713     v << call{handler, arg_regs(2)};
 714     v << jmpi{rh.reenterTC, RegSet()};
 715   });
 716
 717   return us.resumeHelperRet;
 718 }
 719
 720 TCA emitInterpOneCFHelper(CodeBlock& cb, DataBlock& data, Op op,
 721                           const ResumeHelperEntryPoints& rh) {
 722   alignJmpTarget(cb);
 723
 724   return vwrap(cb, data, [&] (Vout& v) {
 725     v << copy2{rvmfp(), rvmsp(), rarg(0), rarg(1)};
 726     // rarg(2) is set at the stub callsite.
 727
 728     auto const handler = reinterpret_cast<TCA>(
 729       interpOneEntryPoints[static_cast<size_t>(op)]
 730     );
 731     v << call{handler, arg_regs(3)};
 732
 733     auto const sf = v.makeReg();
 734     auto const next = v.makeBlock();
 735
 736     v << testq{rret(), rret(), sf};
 737     v << jcci{CC_NZ, sf, next, rh.reenterTC};
 738     v = next;
 739     v << jmpi{rh.resumeHelper};
 740   });
 741 }
 742
 743 void emitInterpOneCFHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us,
 744                             const ResumeHelperEntryPoints& rh,
 745                             const CodeCache& code, Debug::DebugInfo& dbg) {
 746   alignJmpTarget(cb);
 747
 748   auto const emit = [&] (Op op, const char* name) {
 749     auto const stub = emitInterpOneCFHelper(cb, data, op, rh);
 750     us.interpOneCFHelpers[op] = stub;
 751     us.add(name, stub, code, dbg);
 752   };
 753
 754 #define O(name, imm, in, out, flags)          \
 755   if (((flags) & CF) || ((flags) & TF)) {     \
 756     emit(Op::name, "interpOneCFHelper"#name); \
 757   }
 758   OPCODES
 759 #undef O
 760
 761   // Exit is a very special snowflake.  Because it can appear in PHP
 762   // expressions, the emitter pretends that it pushed a value on the eval stack
 763   // (and iopExit actually does push Null right before throwing).  Marking it
 764   // as TF would mess up any bytecodes that want to consume its output value,
 765   // so we can't do that.  But we also don't want to extend regions past it, so
 766   // the JIT treats it as terminal and uses InterpOneCF to execute it.
 767   emit(Op::Exit, "interpOneCFHelperExit");
 768 }
 769
 770 ///////////////////////////////////////////////////////////////////////////////
 771
 772 TCA emitDecRefGeneric(CodeBlock& cb, DataBlock& data) {
 773   CGMeta meta;
 774   alignCacheLine(cb);
 775
 776   auto const start = vwrap(cb, data, meta, [] (Vout& v) {
 777     v << vregrestrict{};
 778     auto const fullFrame = [&] {
 779       switch (arch()) {
 780         case Arch::ARM:
 781         case Arch::PPC64:
 782           return true;
 783         case Arch::X64:
 784           return false;
 785       }
 786       not_reached();
 787     }();
 788     v << stublogue{fullFrame};
 789     if (fullFrame) {
 790       v << copy{rsp(), rvmfp()};
 791     }
 792
 793     auto const rdata = rarg(0);
 794     auto const rtype = rarg(1);
 795
 796     auto const destroy = [&] (Vout& v) {
 797       // decRefGeneric is called via callfaststub, whose ABI claims that all
 798       // registers are preserved.  This is true in the fast path, but in the
 799       // slow path we need to manually save caller-saved registers.
 800       auto const callerSaved = abi().gpUnreserved - abi().calleeSaved;
 801       PhysRegSaver prs{v, callerSaved};
 802
 803       // As a consequence of being called via callfaststub, we can't safely use
 804       // any Vregs here except for status flags registers, at least not with
 805       // the default vwrap() ABI.  Just use the argument registers instead.
 806       assertx(callerSaved.contains(rdata));
 807       assertx(callerSaved.contains(rtype));
 808
 809       auto const dtor = lookupDestructor(v, rtype);
 810       v << callm{dtor, arg_regs(1)};
 811
 812       if (!fullFrame) {
 813         // The stub frame's saved RIP is at %rsp[8] before we saved the
 814         // caller-saved registers.
 815         v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
 816       }
 817     };
 818
 819     emitDecRefWork(v, v, rdata, destroy, false, TRAP_REASON);
 820
 821     v << stubret{{}, fullFrame};
 822   });
 823
 824   meta.process(nullptr);
 825   return start;
 826 }
 827
 828 ///////////////////////////////////////////////////////////////////////////////
 829
 830 TCA emitEnterTCExit(CodeBlock& cb, DataBlock& data, UniqueStubs& /*us*/) {
 831   alignCacheLine(cb);
 832
 833   return vwrap(cb, data, [&] (Vout& v) {
 834     // Eagerly save VM regs.
 835     storeVMRegs(v);
 836
 837     // Realign the native stack.
 838     switch (arch()) {
 839       case Arch::X64:
 840       case Arch::PPC64:
 841         v << lea{rsp()[8], rsp()};
 842         break;
 843       case Arch::ARM:
 844         break;
 845     }
 846
 847     // Store the return value on the top of the eval stack.  Whenever we get to
 848     // enterTCExit, we're semantically executing some PHP construct that sends
 849     // a return value out of a function (either a RetC, or a Yield, or an Await
 850     // that's suspending, etc), and moreover, we must be executing the return
 851     // that leaves this level of VM reentry (i.e. the only way we get here is
 852     // by coming from the callToExit stub or by a phpret{} or leavetc{} that
 853     // undoes the calltc{} or resumetc{} in enterTCHelper).
 854     //
 855     // Either way, we have a live PHP return value in the return registers,
 856     // which we need to put on the top of the evaluation stack.
 857     storeReturnRegs(v);
 858
 859     // Perform a native return.
 860     //
 861     // On PPC64, as there is no new frame created when entering the VM, the FP
 862     // must not be saved.
 863     v << stubret{RegSet(), arch() != Arch::PPC64};
 864   });
 865 }
 866
 867 TCA emitEnterTCHelper(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 868   alignCacheLine(cb);
 869
 870   auto const sp       = rarg(0);
 871   auto const fp       = rarg(1);
 872   auto const start    = rarg(2);
 873   auto const firstAR  = rarg(3);
 874 #ifdef _MSC_VER
 875   auto const tl       = reg::r10;
 876   auto const calleeAR = reg::r11;
 877 #else
 878   auto const tl       = rarg(4);
 879   auto const calleeAR = rarg(5);
 880 #endif
 881
 882   return vwrap2(cb, cb, data, [&] (Vout& v, Vout& vc) {
 883     // Architecture-specific setup for entering the TC.
 884     v << inittc{};
 885
 886     // Native func prologue.
 887     v << stublogue{arch() != Arch::PPC64};
 888
 889 #ifdef _MSC_VER
 890     // Windows hates argument registers.
 891     v << load{rsp()[0x28], reg::r10};
 892     v << load{rsp()[0x30], reg::r11};
 893 #endif
 894
 895     // Set up linkage with the top VM frame in this nesting.
 896     v << store{rsp(), firstAR[AROFF(m_sfp)]};
 897
 898     // Set up the VM registers.
 899     v << copy{fp, rvmfp()};
 900     v << copy{sp, rvmsp()};
 901     v << copy{tl, rvmtl()};
 902
 903     // Unalign the native stack.
 904     switch (arch()) {
 905       case Arch::X64:
 906       case Arch::PPC64:
 907         v << lea{rsp()[-8], rsp()};
 908         break;
 909       case Arch::ARM:
 910         break;
 911     }
 912
 913     // Check if `calleeAR' was set.
 914     auto const sf = v.makeReg();
 915     v << testq{calleeAR, calleeAR, sf};
 916
 917     // We mark this block as unlikely in order to coax the emitter into
 918     // ordering this block last.  This is an important optimization for x64;
 919     // without it, both the jcc for the branch and the jmp for the resumetc{}
 920     // will end up in the same 16-byte extent of code, which messes up the
 921     // branch predictor.
 922     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 923       // No callee means we're resuming in the middle of a TC function.
 924       v << resumetc{start, us.enterTCExit, vm_regs_with_sp()};
 925     });
 926
 927     // We have a callee; set rvmfp() and call it.
 928     v << copy{calleeAR, rvmfp()};
 929     v << calltc{start, rvmfp(), us.enterTCExit, vm_regs_with_sp()};
 930   });
 931 }
 932
 933 TCA emitHandleSRHelper(CodeBlock& cb, DataBlock& data) {
 934   alignCacheLine(cb);
 935
 936   return vwrap(cb, data, [] (Vout& v) {
 937     storeVMRegs(v);
 938
 939     // Pack the service request args into a svcreq::ReqInfo on the stack.
 940     assertx(!(svcreq::kMaxArgs & 1));
 941     for (auto i = svcreq::kMaxArgs; i >= 2; i -= 2) {
 942       v << pushp{r_svcreq_arg(i - 1), r_svcreq_arg(i - 2)};
 943     }
 944     v << pushp{r_svcreq_stub(), r_svcreq_req()};
 945
 946     // Call mcg->handleServiceRequest(rsp()).
 947     auto const sp = v.makeReg();
 948     v << copy{rsp(), sp};
 949
 950     auto const ret = v.makeReg();
 951
 952     v << vcall{
 953       CallSpec::direct(svcreq::handleServiceRequest),
 954       v.makeVcallArgs({{sp}}),
 955       v.makeTuple({ret}),
 956       Fixup{},
 957       DestType::SSA
 958     };
 959
 960     // Pop the ReqInfo off the stack.
 961     auto const reqinfo_sz = static_cast<int>(sizeof(svcreq::ReqInfo));
 962     v << lea{rsp()[reqinfo_sz], rsp()};
 963
 964     // rvmtl() was preserved by the callee, but rvmsp() and rvmfp() might've
 965     // changed if we interpreted anything.  Reload them.  Also load the return
 966     // regs; if we're not returning, it's a spurious load.
 967     loadVMRegs(v);
 968     loadReturnRegs(v);
 969
 970     v << jmpr{ret};
 971   });
 972 }
 973
 974 ///////////////////////////////////////////////////////////////////////////////
 975
 976 TCA emitEndCatchHelper(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 977   alignCacheLine(cb);
 978
 979   auto const udrspo = rvmtl()[unwinderDebuggerReturnSPOff()];
 980
 981   auto const debuggerReturn = vwrap(cb, data, [&] (Vout& v) {
 982     v << load{udrspo, rvmsp()};
 983     v << storeqi{0, udrspo};
 984   });
 985   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_DEBUGGER_RET);
 986
 987   CGMeta meta;
 988
 989   auto const resumeCPPUnwind = vwrap(cb, data, meta, [&] (Vout& v) {
 990     static_assert(sizeof(tl_regState) == 8,
 991                   "The following store must match the size of tl_regState.");
 992     auto const regstate = emitTLSAddr(v, tls_datum(tl_regState));
 993     v << storeqi{static_cast<int32_t>(VMRegState::CLEAN), regstate};
 994
 995     v << load{rvmtl()[unwinderExnOff()], rarg(0)};
 996     v << call{TCA(_Unwind_Resume), arg_regs(1), &us.endCatchHelperPast};
 997     v << trap{TRAP_REASON};
 998   });
 999   meta.process(nullptr);
1000
1001   alignJmpTarget(cb);
1002
1003   return vwrap(cb, data, [&] (Vout& v) {
1004     auto const done1 = v.makeBlock();
1005     auto const sf1 = v.makeReg();
1006
1007     v << cmpqim{0, udrspo, sf1};
1008     v << jcci{CC_NE, sf1, done1, debuggerReturn};
1009     v = done1;
1010
1011     // Normal end catch situation: call back to tc_unwind_resume, which returns
1012     // the catch trace (or null) in the first return register, and the new vmfp
1013     // in the second.
1014     v << copy{rvmfp(), rarg(0)};
1015     v << call{TCA(tc_unwind_resume), arg_regs(1)};
1016     v << copy{rret(1), rvmfp()};
1017
1018     auto const done2 = v.makeBlock();
1019     auto const sf2 = v.makeReg();
1020
1021     v << testq{rret(0), rret(0), sf2};
1022     v << jcci{CC_Z, sf2, done2, resumeCPPUnwind};
1023     v = done2;
1024
1025     v << jmpr{rret(0)};
1026   });
1027 }
1028
1029 TCA emitUnknownExceptionHandler(CodeBlock& cb,
1030                                 DataBlock& data,
1031                                 UniqueStubs& us) {
1032   alignJmpTarget(cb);
1033
1034   CGMeta meta;
1035   auto const ret = vwrap(cb, data, meta, [&] (Vout& v) {
1036     v << call{
1037       TCA(unknownExceptionHandler), {}, &us.unknownExceptionHandlerPast
1038     };
1039   });
1040   meta.process(nullptr);
1041
1042   return ret;
1043 }
1044
1045 TCA emitThrowSwitchMode(CodeBlock& cb, DataBlock& data) {
1046   alignJmpTarget(cb);
1047
1048   return vwrap(cb, data, [] (Vout& v) {
1049     v << call{TCA(throwSwitchMode)};
1050     v << trap{TRAP_REASON};
1051   });
1052 }
1053
1054 template<class F>
1055 TCA emitHelperThunk(CodeCache& code, CodeBlock& cb, DataBlock& data, F* func) {
1056   // we only emit these calls into hot, main and cold.
1057   if (deltaFits(code.base() - (TCA)func, sz::dword) &&
1058       deltaFits(code.frozen().base() - (TCA)func, sz::dword)) {
1059     return (TCA)func;
1060   }
1061   alignJmpTarget(cb);
1062   return vwrap(cb, data, [&] (Vout& v) {
1063       v << jmpi{(TCA)func};
1064   });
1065 }
1066
1067 ///////////////////////////////////////////////////////////////////////////////
1068
1069 }
1070
1071 void UniqueStubs::emitAll(CodeCache& code, Debug::DebugInfo& dbg) {
1072   auto view = code.view();
1073   auto& main = view.main();
1074   auto& cold = view.cold();
1075   auto& frozen = view.frozen();
1076   auto& hotBlock = code.view(TransKind::Optimize).main();
1077   auto& data = view.data();
1078
1079   auto const hot = [&]() -> CodeBlock& {
1080     return hotBlock.available() > 512 ? hotBlock : main;
1081   };
1082
1083 #define ADD(name, stub) name = add(#name, (stub), code, dbg)
1084   ADD(enterTCExit,   emitEnterTCExit(main, data, *this));
1085   enterTCHelper =
1086     decltype(enterTCHelper)(add("enterTCHelper",
1087                                 emitEnterTCHelper(main, data, *this),
1088                                 code,
1089                                 dbg));
1090
1091   // These guys are required by a number of other stubs.
1092   ADD(handleSRHelper, emitHandleSRHelper(hot(), data));
1093   ADD(endCatchHelper, emitEndCatchHelper(hot(), data, *this));
1094   ADD(unknownExceptionHandler, emitUnknownExceptionHandler(cold, data, *this));
1095
1096   ADD(funcPrologueRedispatch, emitFuncPrologueRedispatch(hot(), data));
1097   ADD(fcallHelperThunk,       emitFCallHelperThunk(cold, frozen, data));
1098   ADD(funcBodyHelperThunk,    emitFuncBodyHelperThunk(cold, data));
1099   ADD(functionEnterHelper, emitFunctionEnterHelper(cold, frozen, data, *this));
1100   ADD(functionSurprisedOrStackOverflow,
1101       emitFunctionSurprisedOrStackOverflow(cold, frozen, data, *this));
1102
1103   ADD(retHelper,                  emitInterpRet(hot(), data));
1104   ADD(genRetHelper,               emitInterpGenRet<false>(cold, data));
1105   ADD(asyncGenRetHelper,          emitInterpGenRet<true>(hot(), data));
1106   ADD(retInlHelper,               emitInterpRet(hot(), data));
1107   ADD(debuggerRetHelper,          emitDebuggerInterpRet(cold, data));
1108   ADD(debuggerGenRetHelper,       emitDebuggerInterpGenRet<false>(cold, data));
1109   ADD(debuggerAsyncGenRetHelper,  emitDebuggerInterpGenRet<true>(cold, data));
1110
1111   ADD(bindCallStub,          emitBindCallStub<false>(cold, data));
1112   ADD(immutableBindCallStub, emitBindCallStub<true>(cold, data));
1113   ADD(fcallUnpackHelper,     emitFCallUnpackHelper(hot(), cold, data, *this));
1114
1115   ADD(decRefGeneric,  emitDecRefGeneric(hot(), data));
1116
1117   ADD(callToExit,         emitCallToExit(hot(), data, *this));
1118   ADD(throwSwitchMode,    emitThrowSwitchMode(frozen, data));
1119
1120   ADD(handlePrimeCacheInit,
1121       emitHelperThunk(code, cold, data,
1122                       MethodCache::handlePrimeCacheInit<false>));
1123   ADD(handlePrimeCacheInitFatal,
1124       emitHelperThunk(code, cold, data,
1125                       MethodCache::handlePrimeCacheInit<true>));
1126   ADD(handleSlowPath,
1127       emitHelperThunk(code, main, data,
1128                       MethodCache::handleSlowPath<false>));
1129   ADD(handleSlowPathFatal,
1130       emitHelperThunk(code, main, data,
1131                       MethodCache::handleSlowPath<true>));
1132
1133 #undef ADD
1134
1135   add("freeLocalsHelpers",
1136       emitFreeLocalsHelpers(hot(), data, *this), code, dbg);
1137
1138   ResumeHelperEntryPoints rh;
1139   add("resumeInterpHelpers",
1140       emitResumeInterpHelpers(hot(), data, *this, rh),
1141       code, dbg);
1142   emitInterpOneCFHelpers(cold, data, *this, rh, code, dbg);
1143
1144   emitAllResumable(code, dbg);
1145 }
1146
1147 ///////////////////////////////////////////////////////////////////////////////
1148
1149 TCA UniqueStubs::add(const char* name, TCA start,
1150                      const CodeCache& code, Debug::DebugInfo& dbg) {
1151   if (!code.isValidCodeAddress(start)) return start;
1152
1153   auto& cb = code.blockFor(start);
1154   auto const end = cb.frontier();
1155
1156   FTRACE(1, "unique stub: {} @ {} -- {:4} bytes: {}\n",
1157          cb.name(),
1158          static_cast<void*>(start),
1159          static_cast<size_t>(end - start),
1160          name);
1161
1162   ONTRACE(2,
1163           [&]{
1164             std::ostringstream os;
1165             disasmRange(os, start, end);
1166             FTRACE(2, "{}\n", os.str());
1167           }()
1168          );
1169
1170   if (!RuntimeOption::EvalJitNoGdb) {
1171     dbg.recordStub(Debug::TCRange(start, end, &cb == &code.cold()),
1172                    folly::sformat("HHVM::{}", name));
1173   }
1174   if (RuntimeOption::EvalJitUseVtuneAPI) {
1175     reportHelperToVtune(folly::sformat("HHVM::{}", name).c_str(),
1176                         start,
1177                         end);
1178   }
1179   if (RuntimeOption::EvalPerfPidMap) {
1180     dbg.recordPerfMap(Debug::TCRange(start, end, &cb == &code.cold()),
1181                       SrcKey{},
1182                       nullptr,
1183                       false,
1184                       false,
1185                       folly::sformat("HHVM::{}", name));
1186   }
1187
1188   auto const newStub = StubRange{name, start, end};
1189   auto lower = std::lower_bound(m_ranges.begin(), m_ranges.end(), newStub);
1190
1191   // We assume ranges are non-overlapping.
1192   assertx(lower == m_ranges.end() || newStub.end <= lower->start);
1193   assertx(lower == m_ranges.begin() || (lower - 1)->end <= newStub.start);
1194   m_ranges.insert(lower, newStub);
1195   return start;
1196 }
1197
1198 std::string UniqueStubs::describe(TCA address) const {
1199   auto raw = [address] { return folly::sformat("{}", address); };
1200   if (m_ranges.empty()) return raw();
1201
1202   auto const dummy = StubRange{"", address, nullptr};
1203   auto lower = std::upper_bound(m_ranges.begin(), m_ranges.end(), dummy);
1204   if (lower == m_ranges.begin()) return raw();
1205
1206   --lower;
1207   if (lower->contains(address)) {
1208     return folly::sformat("{}+{:#x}", lower->name, address - lower->start);
1209   }
1210   return raw();
1211 }
1212
1213 ///////////////////////////////////////////////////////////////////////////////
1214
1215 RegSet interp_one_cf_regs() {
1216   return vm_regs_with_sp() | rarg(2);
1217 }
1218
1219 void emitInterpReq(Vout& v, SrcKey sk, FPInvOffset spOff) {
1220   if (sk.resumeMode() == ResumeMode::None) {
1221     v << lea{rvmfp()[-cellsToBytes(spOff.offset)], rvmsp()};
1222   }
1223   v << copy{v.cns(sk.pc()), rarg(0)};
1224   v << jmpi{tc::ustubs().interpHelper, arg_regs(1)};
1225 }
1226
1227 ///////////////////////////////////////////////////////////////////////////////
1228
1229 void enterTCImpl(TCA start, ActRec* stashedAR) {
1230   // We have to force C++ to spill anything that might be in a callee-saved
1231   // register (aside from rvmfp()), since enterTCHelper does not save them.
1232   CALLEE_SAVED_BARRIER();
1233   auto& regs = vmRegsUnsafe();
1234   tc::ustubs().enterTCHelper(regs.stack.top(), regs.fp, start,
1235                               vmFirstAR(), rds::tl_base, stashedAR);
1236   CALLEE_SAVED_BARRIER();
1237 }
1238
1239 ///////////////////////////////////////////////////////////////////////////////
1240
1241 }}