hphp/runtime/vm/jit/unique-stubs.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/unique-stubs.h"
  18
  19 #include "hphp/runtime/base/datatype.h"
  20 #include "hphp/runtime/base/rds-header.h"
  21 #include "hphp/runtime/base/runtime-option.h"
  22 #include "hphp/runtime/base/surprise-flags.h"
  23 #include "hphp/runtime/base/tv-mutate.h"
  24 #include "hphp/runtime/base/tv-variant.h"
  25 #include "hphp/runtime/vm/bytecode.h"
  26 #include "hphp/runtime/vm/debug/debug.h"
  27 #include "hphp/runtime/vm/event-hook.h"
  28 #include "hphp/runtime/vm/hhbc.h"
  29 #include "hphp/runtime/vm/interp-helpers.h"
  30 #include "hphp/runtime/vm/srckey.h"
  31 #include "hphp/runtime/vm/vm-regs.h"
  32
  33 #include "hphp/runtime/vm/jit/types.h"
  34 #include "hphp/runtime/vm/jit/abi.h"
  35 #include "hphp/runtime/vm/jit/align.h"
  36 #include "hphp/runtime/vm/jit/cg-meta.h"
  37 #include "hphp/runtime/vm/jit/code-cache.h"
  38 #include "hphp/runtime/vm/jit/code-gen-cf.h"
  39 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  40 #include "hphp/runtime/vm/jit/code-gen-tls.h"
  41 #include "hphp/runtime/vm/jit/debugger.h"
  42 #include "hphp/runtime/vm/jit/fixup.h"
  43 #include "hphp/runtime/vm/jit/mcgen.h"
  44 #include "hphp/runtime/vm/jit/phys-reg.h"
  45 #include "hphp/runtime/vm/jit/phys-reg-saver.h"
  46 #include "hphp/runtime/vm/jit/print.h"
  47 #include "hphp/runtime/vm/jit/service-request-handlers.h"
  48 #include "hphp/runtime/vm/jit/service-requests.h"
  49 #include "hphp/runtime/vm/jit/smashable-instr.h"
  50 #include "hphp/runtime/vm/jit/stack-offsets.h"
  51 #include "hphp/runtime/vm/jit/stack-overflow.h"
  52 #include "hphp/runtime/vm/jit/target-cache.h"
  53 #include "hphp/runtime/vm/jit/tc.h"
  54 #include "hphp/runtime/vm/jit/translator-inline.h"
  55 #include "hphp/runtime/vm/jit/unique-stubs-arm.h"
  56 #include "hphp/runtime/vm/jit/unique-stubs-ppc64.h"
  57 #include "hphp/runtime/vm/jit/unique-stubs-x64.h"
  58 #include "hphp/runtime/vm/jit/unwind-itanium.h"
  59 #include "hphp/runtime/vm/jit/vasm-gen.h"
  60 #include "hphp/runtime/vm/jit/vasm-instr.h"
  61 #include "hphp/runtime/vm/jit/vasm-reg.h"
  62 #include "hphp/runtime/vm/jit/vtune-jit.h"
  63
  64 #include "hphp/runtime/ext/asio/ext_async-generator.h"
  65 #include "hphp/runtime/ext/generator/ext_generator.h"
  66
  67 #include "hphp/util/abi-cxx.h"
  68 #include "hphp/util/arch.h"
  69 #include "hphp/util/asm-x64.h"
  70 #include "hphp/util/data-block.h"
  71 #include "hphp/util/trace.h"
  72
  73 #include <folly/Format.h>
  74
  75 #include <algorithm>
  76
  77 namespace HPHP { namespace jit {
  78
  79 ///////////////////////////////////////////////////////////////////////////////
  80
  81 TRACE_SET_MOD(ustubs);
  82
  83 ///////////////////////////////////////////////////////////////////////////////
  84
  85 namespace {
  86
  87 ///////////////////////////////////////////////////////////////////////////////
  88
  89 void alignJmpTarget(CodeBlock& cb) {
  90   align(cb, nullptr, Alignment::JmpTarget, AlignContext::Dead);
  91 }
  92
  93 void assertNativeStackAligned(Vout& v) {
  94   if (RuntimeOption::EvalHHIRGenerateAsserts) {
  95     v << call{TCA(assert_native_stack_aligned)};
  96   }
  97 }
  98
  99 /*
 100  * Load and store the VM registers from/to RDS.
 101  */
 102 void loadVMRegs(Vout& v) {
 103   v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 104   v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 105 }
 106 void storeVMRegs(Vout& v) {
 107   v << store{rvmfp(), rvmtl()[rds::kVmfpOff]};
 108   v << store{rvmsp(), rvmtl()[rds::kVmspOff]};
 109 }
 110
 111 /*
 112  * Load and store the PHP return registers from/to the top of the VM stack.
 113  *
 114  * Note that we don't do loadb{}/storeb{} for the type register, because we
 115  * sometimes need to preserve the m_aux field across returns.
 116  */
 117 void loadReturnRegs(Vout& v) {
 118   v << load{rvmsp()[TVOFF(m_data)], rret_data()};
 119   v << load{rvmsp()[TVOFF(m_type)], rret_type()};
 120 }
 121 void storeReturnRegs(Vout& v) {
 122   v << store{rret_data(), rvmsp()[TVOFF(m_data)]};
 123   v << store{rret_type(), rvmsp()[TVOFF(m_type)]};
 124 }
 125
 126 /*
 127  * Convenience wrapper around a simple vcall to `helper', with a single `arg'
 128  * and a return value in `d'.
 129  */
 130 template<class F>
 131 Vinstr simplecall(Vout& v, F helper, Vreg arg, Vreg d) {
 132   return vcall{
 133     CallSpec::direct(helper),
 134     v.makeVcallArgs({{arg}}),
 135     v.makeTuple({d}),
 136     Fixup{},
 137     DestType::SSA
 138   };
 139 }
 140
 141 /*
 142  * Convenience wrapper around a simple vcall to `helper', with a single `arg'
 143  * and a pair of return values in `d1' and `d2'.
 144  */
 145 template<class F>
 146 Vinstr simplecall(Vout& v, F helper, Vreg arg, Vreg d1, Vreg d2) {
 147   return vcall{
 148     CallSpec::direct(helper),
 149     v.makeVcallArgs({{arg}}),
 150     v.makeTuple({d1, d2}),
 151     Fixup{},
 152     DestType::SSAPair
 153   };
 154 }
 155
 156 /*
 157  * Emit a catch trace that unwinds a stub context back to the PHP context that
 158  * called it.
 159  */
 160 template<class GenFn>
 161 void emitStubCatch(Vout& v, const UniqueStubs& us, GenFn gen) {
 162   always_assert(us.endCatchHelper);
 163   v << landingpad{};
 164   gen(v);
 165   v << stubunwind{};
 166   v << jmpi{us.endCatchHelper};
 167 }
 168
 169 ///////////////////////////////////////////////////////////////////////////////
 170
 171 TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 172   return ARCH_SWITCH_CALL(emitFreeLocalsHelpers, cb, data, us);
 173 }
 174
 175 TCA emitCallToExit(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 176   return ARCH_SWITCH_CALL(emitCallToExit, cb, data, us);
 177 }
 178
 179 ///////////////////////////////////////////////////////////////////////////////
 180
 181 struct FCallHelperRet {
 182   TCA destAddr;
 183   TCA retAddr;
 184 };
 185
 186 FCallHelperRet fcallHelper(ActRec* ar) {
 187   assert_native_stack_aligned();
 188   assertx(!ar->resumed());
 189
 190   if (LIKELY(!RuntimeOption::EvalFailJitPrologs)) {
 191     auto const tca = mcgen::getFuncPrologue(
 192       const_cast<Func*>(ar->func()),
 193       ar->numArgs()
 194     );
 195     if (tca) return { tca, nullptr };
 196   }
 197
 198   // Check for stack overflow in the same place func prologues make their
 199   // StackCheck::Early check (see irgen-func-prologue.cpp).  This handler also
 200   // cleans and syncs vmRegs for us.
 201   if (checkCalleeStackOverflow(ar)) handleStackOverflow(ar);
 202
 203   // If doFCall indicates that the function was intercepted and should be
 204   // skipped, it will have already torn down the callee's frame. So, we need to
 205   // save the return value thats in it.
 206   auto const retAddr = (TCA)ar->m_savedRip;
 207
 208   try {
 209     VMRegAnchor _(ar);
 210     if (doFCall(ar, vmpc())) {
 211       return { tc::ustubs().resumeHelperRet, nullptr };
 212     }
 213     // We've been asked to skip the function body (fb_intercept).  The vmregs
 214     // have already been fixed; indicate this with a nullptr return.
 215     return { nullptr, retAddr };
 216   } catch (...) {
 217     // The VMRegAnchor above took care of us, but we need to tell the unwinder
 218     // (since ~VMRegAnchor() will have reset tl_regState).
 219     tl_regState = VMRegState::CLEAN;
 220     throw;
 221   }
 222 }
 223
 224 ///////////////////////////////////////////////////////////////////////////////
 225
 226 TCA emitFuncPrologueRedispatch(CodeBlock& cb, DataBlock& data) {
 227   alignJmpTarget(cb);
 228
 229   return vwrap(cb, data, [] (Vout& v) {
 230     auto const func = v.makeReg();
 231     v << load{rvmfp()[AROFF(m_func)], func};
 232
 233     auto const argc = v.makeReg();
 234     auto const naaf = v.makeReg();
 235     v << loadl{rvmfp()[AROFF(m_numArgsAndFlags)], naaf};
 236     v << andli{ActRec::kNumArgsMask, naaf, argc, v.makeReg()};
 237
 238     auto const nparams = v.makeReg();
 239     auto const pcounts = v.makeReg();
 240     v << loadl{func[Func::paramCountsOff()], pcounts};
 241     v << shrli{0x1, pcounts, nparams, v.makeReg()};
 242
 243     auto const sf = v.makeReg();
 244     v << cmpl{argc, nparams, sf};
 245
 246     auto const pTabOff = safe_cast<int32_t>(Func::prologueTableOff());
 247     auto const ptrSize = safe_cast<int32_t>(sizeof(LowPtr<uint8_t>));
 248
 249     // If we passed more args than declared, we need to dispatch to the
 250     // "too many arguments" prologue.
 251     ifThen(v, CC_L, sf, [&] (Vout& v) {
 252       auto const dest = v.makeReg();
 253
 254       auto const nargs = v.makeReg();
 255       v << movzlq{nparams, nargs};
 256
 257       emitLdLowPtr(v, func[nargs * ptrSize + (pTabOff + ptrSize)],
 258                    dest, sizeof(LowPtr<uint8_t>));
 259       v << jmpr{dest};
 260     });
 261
 262     auto const nargs = v.makeReg();
 263     v << movzlq{argc, nargs};
 264
 265     auto const dest = v.makeReg();
 266     emitLdLowPtr(v, func[nargs * ptrSize + pTabOff],
 267                  dest, sizeof(LowPtr<uint8_t>));
 268     v << jmpr{dest};
 269   });
 270 }
 271
 272 TCA emitFCallHelperThunk(CodeBlock& main, CodeBlock& cold, DataBlock& data) {
 273   alignJmpTarget(main);
 274
 275   return vwrap2(main, cold, data, [] (Vout& v, Vout& vc) {
 276     v << phplogue{rvmfp()};
 277
 278     // fcallHelper asserts native stack alignment for us.
 279     FCallHelperRet (*helper)(ActRec*) = &fcallHelper;
 280     auto const dest = v.makeReg();
 281     auto const saved_rip = v.makeReg();
 282     v << simplecall(v, helper, rvmfp(), dest, saved_rip);
 283
 284     // Clobber rvmsp in debug builds.
 285     if (debug) v << copy{v.cns(0x1), rvmsp()};
 286
 287     auto const sf = v.makeReg();
 288     v << testq{dest, dest, sf};
 289
 290     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 291       // A nullptr dest means the callee was intercepted and should be
 292       // skipped. In that case, saved_rip will contain the return address that
 293       // was in the callee's ActRec before it was torn down by the intercept.
 294       loadVMRegs(v);
 295       loadReturnRegs(v);
 296
 297       // Return to the caller. This unbalances the return stack buffer, but if
 298       // we're intercepting, we probably don't care.
 299       v << jmpr{saved_rip};
 300     });
 301
 302     // Jump to the func prologue.
 303     v << tailcallphp{dest, rvmfp(), php_call_regs()};
 304   });
 305 }
 306
 307 TCA emitFuncBodyHelperThunk(CodeBlock& cb, DataBlock& data) {
 308   alignJmpTarget(cb);
 309
 310   return vwrap(cb, data, [] (Vout& v) {
 311     TCA (*helper)(ActRec*) = &svcreq::funcBodyHelper;
 312     auto const dest = v.makeReg();
 313     v << simplecall(v, helper, rvmfp(), dest);
 314     v << jmpr{dest};
 315   });
 316 }
 317
 318 TCA emitFunctionEnterHelper(CodeBlock& main, CodeBlock& cold,
 319                             DataBlock& data, UniqueStubs& us) {
 320   alignJmpTarget(main);
 321
 322   CGMeta meta;
 323
 324   auto const start = vwrap2(main, cold, data, meta, [&] (Vout& v, Vout& vc) {
 325     auto const ar = v.makeReg();
 326
 327     v << copy{rvmfp(), ar};
 328
 329     // Fully set up the call frame for the stub.  We can't skip this like we do
 330     // in other stubs because we need the return IP for this frame in the %rbp
 331     // chain, in order to find the proper fixup for the VMRegAnchor in the
 332     // intercept handler.
 333     v << stublogue{true};
 334     v << copy{rsp(), rvmfp()};
 335
 336     // When we call the event hook, it might tell us to skip the callee
 337     // (because of fb_intercept).  If that happens, we need to return to the
 338     // caller, but the handler will have already popped the callee's frame.
 339     // So, we need to save these values for later.
 340     v << pushpm{ar[AROFF(m_savedRip)], ar[AROFF(m_sfp)]};
 341
 342     v << copy2{ar, v.cns(EventHook::NormalFunc), rarg(0), rarg(1)};
 343
 344     auto const done = v.makeBlock();
 345     auto const ctch = vc.makeBlock();
 346     auto const should_continue = v.makeReg();
 347     bool (*hook)(const ActRec*, int) = &EventHook::onFunctionCall;
 348
 349     v << vinvoke{
 350       CallSpec::direct(hook),
 351       v.makeVcallArgs({{ar, v.cns(EventHook::NormalFunc)}}),
 352       v.makeTuple({should_continue}),
 353       {done, ctch},
 354       Fixup{},
 355       DestType::SSA
 356     };
 357
 358     vc = ctch;
 359     emitStubCatch(vc, us, [] (Vout& v) {
 360       // Skip past the stuff we saved for the intercept case.
 361       v << lea{rsp()[16], rsp()};
 362       // Undo our stub frame, so that rvmfp() points to the parent VM frame.
 363       v << load{rsp()[AROFF(m_sfp)], rvmfp()};
 364     });
 365
 366     v = done;
 367
 368     auto const sf = v.makeReg();
 369     v << testb{should_continue, should_continue, sf};
 370
 371     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 372       auto const saved_rip = v.makeReg();
 373
 374       // The event hook has already cleaned up the stack and popped the
 375       // callee's frame, so we're ready to continue from the original call
 376       // site.  We just need to grab the fp/rip of the original frame that we
 377       // saved earlier, and sync rvmsp().
 378       v << popp{rvmfp(), saved_rip};
 379
 380       // Drop our call frame; the stublogue{} instruction guarantees that this
 381       // is exactly 16 bytes.
 382       v << lea{rsp()[kNativeFrameSize], rsp()};
 383
 384       // Sync vmsp and the return regs.
 385       v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 386       v << load{rvmsp()[TVOFF(m_data)], rret_data()};
 387       v << load{rvmsp()[TVOFF(m_type)], rret_type()};
 388
 389       // Return to the caller.  This unbalances the return stack buffer, but if
 390       // we're intercepting, we probably don't care.
 391       v << jmpr{saved_rip};
 392     });
 393
 394     // Skip past the stuff we saved for the intercept case.
 395     v << lea{rsp()[16], rsp()};
 396
 397     // Restore rvmfp() and return to the callee's func prologue.
 398     v << stubret{RegSet(), true};
 399   });
 400
 401   meta.process(nullptr);
 402   return start;
 403 }
 404
 405 TCA emitFunctionSurprisedOrStackOverflow(CodeBlock& main,
 406                                          CodeBlock& cold,
 407                                          DataBlock& data,
 408                                          const UniqueStubs& us) {
 409   alignJmpTarget(main);
 410
 411   CGMeta meta;
 412
 413   auto const start = vwrap2(main, cold, data, meta, [&] (Vout& v, Vout& vc) {
 414     v << stublogue{};
 415
 416     auto const done = v.makeBlock();
 417     auto const ctch = vc.makeBlock();
 418
 419     v << vinvoke{CallSpec::direct(handlePossibleStackOverflow),
 420                  v.makeVcallArgs({{rvmfp()}}), v.makeTuple({}),
 421                  {done, ctch}};
 422     vc = ctch;
 423     emitStubCatch(vc, us, [](Vout& /*v*/) {});
 424
 425     v = done;
 426     v << tailcallstub{us.functionEnterHelper};
 427   });
 428
 429   meta.process(nullptr);
 430   return start;
 431 }
 432
 433 ///////////////////////////////////////////////////////////////////////////////
 434
 435 template<bool async>
 436 void loadGenFrame(Vout& v, Vreg d) {
 437   auto const arOff = BaseGenerator::arOff() -
 438     (async ? AsyncGenerator::objectOff() : Generator::objectOff());
 439
 440   auto const gen = v.makeReg();
 441
 442   // We have to get the Generator object from the current frame's $this, then
 443   // load the embedded frame.
 444   v << load{rvmfp()[AROFF(m_thisUnsafe)], gen};
 445   v << lea{gen[arOff], d};
 446 }
 447
 448 void debuggerRetImpl(Vout& v, Vreg ar) {
 449   auto const soff = v.makeReg();
 450
 451   v << loadl{ar[AROFF(m_soff)], soff};
 452   v << storel{soff, rvmtl()[unwinderDebuggerReturnOffOff()]};
 453   v << store{rvmsp(), rvmtl()[unwinderDebuggerReturnSPOff()]};
 454
 455   auto const ret = v.makeReg();
 456   v << simplecall(v, unstashDebuggerCatch, ar, ret);
 457
 458   v << jmpr{ret};
 459 }
 460
 461 TCA emitInterpRet(CodeBlock& cb, DataBlock& data) {
 462   alignJmpTarget(cb);
 463
 464   auto const start = vwrap(cb, data, [] (Vout& v) {
 465     // Sync return regs before calling native assert function.
 466     storeReturnRegs(v);
 467     assertNativeStackAligned(v);
 468
 469     v << lea{rvmsp()[-kArRetOff], r_svcreq_arg(0)};
 470     v << copy{rvmfp(), r_svcreq_arg(1)};
 471   });
 472   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_INTERP_RET);
 473   return start;
 474 }
 475
 476 template<bool async>
 477 TCA emitInterpGenRet(CodeBlock& cb, DataBlock& data) {
 478   alignJmpTarget(cb);
 479
 480   auto const start = vwrap(cb, data, [] (Vout& v) {
 481     // Sync return regs before calling native assert function.
 482     storeReturnRegs(v);
 483     assertNativeStackAligned(v);
 484
 485     loadGenFrame<async>(v, r_svcreq_arg(0));
 486     v << copy{rvmfp(), r_svcreq_arg(1)};
 487   });
 488   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_INTERP_RET);
 489   return start;
 490 }
 491
 492 TCA emitDebuggerInterpRet(CodeBlock& cb, DataBlock& data) {
 493   alignJmpTarget(cb);
 494
 495   return vwrap(cb, data, [] (Vout& v) {
 496     // Sync return regs before calling native assert function.
 497     storeReturnRegs(v);
 498     assertNativeStackAligned(v);
 499
 500     auto const ar = v.makeReg();
 501     v << lea{rvmsp()[-kArRetOff], ar};
 502     debuggerRetImpl(v, ar);
 503   });
 504 }
 505
 506 template<bool async>
 507 TCA emitDebuggerInterpGenRet(CodeBlock& cb, DataBlock& data) {
 508   alignJmpTarget(cb);
 509
 510   return vwrap(cb, data, [] (Vout& v) {
 511     assertNativeStackAligned(v);
 512
 513     auto const ar = v.makeReg();
 514     loadGenFrame<async>(v, ar);
 515     debuggerRetImpl(v, ar);
 516   });
 517 }
 518
 519 ///////////////////////////////////////////////////////////////////////////////
 520
 521 template<bool immutable>
 522 TCA emitBindCallStub(CodeBlock& cb, DataBlock& data) {
 523   return vwrap(cb, data, [] (Vout& v) {
 524     v << phplogue{rvmfp()};
 525
 526     auto args = VregList { v.makeReg(), v.makeReg(), v.makeReg() };
 527
 528     // Reconstruct the address of the call from the saved RIP.
 529     auto const savedRIP = v.makeReg();
 530     auto const callLen = safe_cast<int>(smashableCallLen());
 531     v << load{rvmfp()[AROFF(m_savedRip)], savedRIP};
 532     v << subqi{callLen, savedRIP, args[0], v.makeReg()};
 533
 534     v << copy{rvmfp(), args[1]};
 535     v << movb{v.cns(immutable), args[2]};
 536
 537     auto const ret = v.makeReg();
 538
 539     v << vcall{
 540       CallSpec::direct(svcreq::handleBindCall),
 541       v.makeVcallArgs({args}),
 542       v.makeTuple({ret}),
 543       Fixup{},
 544       DestType::SSA
 545     };
 546
 547     v << tailcallphp{ret, rvmfp(), php_call_regs()};
 548   });
 549 }
 550
 551 TCA emitFCallArrayHelper(CodeBlock& main, CodeBlock& cold,
 552                          DataBlock& data, UniqueStubs& us) {
 553   align(main, nullptr, Alignment::CacheLine, AlignContext::Dead);
 554
 555   CGMeta meta;
 556
 557   auto const ret = vwrap(main, data, [] (Vout& v) {
 558     v << movl{v.cns(0), rarg(2)};
 559   });
 560
 561   us.fcallUnpackHelper = vwrap2(main, cold, data, meta,
 562                                 [&] (Vout& v, Vout& vc) {
 563     // We reach fcallArrayHelper in the same context as a func prologue, so
 564     // this should really be a phplogue{}---but we don't need the return
 565     // address in the ActRec until later, and in the event the callee is
 566     // intercepted, we must save it on the stack because the callee frame will
 567     // already have been popped.  So use a stublogue and "convert" it manually
 568     // later.
 569     v << stublogue{};
 570
 571     storeVMRegs(v);
 572
 573     auto const func = v.makeReg();
 574     auto const unit = v.makeReg();
 575     auto const bc = v.makeReg();
 576
 577     // Load fp->m_func->m_unit->m_bc.
 578     v << load{rvmfp()[AROFF(m_func)], func};
 579     v << load{func[Func::unitOff()], unit};
 580     v << load{unit[Unit::bcOff()], bc};
 581
 582     auto const pc = v.makeReg();
 583     auto const next = v.makeReg();
 584
 585     // Convert offsets into PCs, and sync the PC.
 586     v << addq{bc, rarg(0), pc, v.makeReg()};
 587     v << store{pc, rvmtl()[rds::kVmpcOff]};
 588     v << addq{bc, rarg(1), next, v.makeReg()};
 589
 590     auto const retAddr = v.makeReg();
 591     v << loadstubret{retAddr};
 592
 593     auto const done = v.makeBlock();
 594     auto const ctch = vc.makeBlock();
 595     auto const should_continue = v.makeReg();
 596     bool (*helper)(PC, int32_t, void*) = &doFCallArrayTC;
 597
 598     v << vinvoke{
 599       CallSpec::direct(helper),
 600       v.makeVcallArgs({{next, rarg(2), retAddr}}),
 601       v.makeTuple({should_continue}),
 602       {done, ctch},
 603       Fixup{},
 604       DestType::SSA
 605     };
 606     vc = ctch;
 607     emitStubCatch(vc, us, [] (Vout& v) { loadVMRegs(v); });
 608
 609     v = done;
 610
 611     // Load only rvmsp(); we need to wait to make sure we aren't skipping the
 612     // callee before loading rvmfp().
 613     v << load{rvmtl()[rds::kVmspOff], rvmsp()};
 614
 615     auto const sf = v.makeReg();
 616     v << testb{should_continue, should_continue, sf};
 617
 618     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 619       // If false was returned, we should skip the callee.  The interpreter
 620       // will have popped the pre-live ActRec already, so we can just return to
 621       // the caller after syncing the return regs.
 622       loadReturnRegs(v);
 623       v << stubret{};
 624     });
 625     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 626
 627     // If true was returned, we're calling the callee, so undo the stublogue{}
 628     // and convert to a phplogue{}.
 629     v << stubtophp{rvmfp()};
 630
 631     auto const callee = v.makeReg();
 632     auto const body = v.makeReg();
 633
 634     v << load{rvmfp()[AROFF(m_func)], callee};
 635     emitLdLowPtr(v, callee[Func::funcBodyOff()], body, sizeof(LowPtr<uint8_t>));
 636
 637     // We jmp directly to the func body---this keeps the return stack buffer
 638     // balanced between the call to this stub and the ret from the callee.
 639     v << jmpr{body};
 640   });
 641
 642   meta.process(nullptr);
 643   return ret;
 644 }
 645
 646 ///////////////////////////////////////////////////////////////////////////////
 647
 648 struct ResumeHelperEntryPoints {
 649   TCA resumeHelperRet;
 650   TCA resumeHelper;
 651   TCA handleResume;
 652   TCA reenterTC;
 653 };
 654
 655 ResumeHelperEntryPoints emitResumeHelpers(CodeBlock& cb, DataBlock& data) {
 656   ResumeHelperEntryPoints rh;
 657
 658   rh.resumeHelperRet = vwrap(cb, data, [] (Vout& v) {
 659     v << phplogue{rvmfp()};
 660   });
 661   rh.resumeHelper = vwrap(cb, data, [] (Vout& v) {
 662     v << ldimmb{0, rarg(0)};
 663   });
 664
 665   rh.handleResume = vwrap(cb, data, [] (Vout& v) {
 666     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 667
 668     auto const handler = reinterpret_cast<TCA>(svcreq::handleResume);
 669     v << call{handler, arg_regs(2)};
 670   });
 671
 672   rh.reenterTC = vwrap(cb, data, [] (Vout& v) {
 673     // Save the return of handleResume(), then sync regs.
 674     auto const target = v.makeReg();
 675     v << copy{rret(), target};
 676
 677     loadVMRegs(v);
 678     loadReturnRegs(v);  // spurious load if we're not returning
 679
 680     v << jmpr{target};
 681   });
 682
 683   return rh;
 684 }
 685
 686 TCA emitResumeInterpHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us,
 687                             ResumeHelperEntryPoints& rh) {
 688   alignJmpTarget(cb);
 689
 690   rh = emitResumeHelpers(cb, data);
 691
 692   us.resumeHelperRet = rh.resumeHelperRet;
 693   us.resumeHelper = rh.resumeHelper;
 694
 695   us.interpHelper = vwrap(cb, data, [] (Vout& v) {
 696     v << store{rarg(0), rvmtl()[rds::kVmpcOff]};
 697   });
 698   us.interpHelperSyncedPC = vwrap(cb, data, [&] (Vout& v) {
 699     storeVMRegs(v);
 700     v << ldimmb{1, rarg(0)};
 701     v << jmpi{rh.handleResume, RegSet(rarg(0))};
 702   });
 703
 704   us.fcallAwaitSuspendHelper = vwrap(cb, data, [&] (Vout& v) {
 705     v << load{rvmtl()[rds::kVmfpOff], rvmfp()};
 706
 707     auto const handler = reinterpret_cast<TCA>(svcreq::handleFCallAwaitSuspend);
 708     v << call{handler, arg_regs(2)};
 709     v << jmpi{rh.reenterTC, RegSet()};
 710   });
 711
 712   return us.resumeHelperRet;
 713 }
 714
 715 TCA emitInterpOneCFHelper(CodeBlock& cb, DataBlock& data, Op op,
 716                           const ResumeHelperEntryPoints& rh) {
 717   alignJmpTarget(cb);
 718
 719   return vwrap(cb, data, [&] (Vout& v) {
 720     v << copy2{rvmfp(), rvmsp(), rarg(0), rarg(1)};
 721     // rarg(2) is set at the stub callsite.
 722
 723     auto const handler = reinterpret_cast<TCA>(
 724       interpOneEntryPoints[static_cast<size_t>(op)]
 725     );
 726     v << call{handler, arg_regs(3)};
 727
 728     auto const sf = v.makeReg();
 729     auto const next = v.makeBlock();
 730
 731     v << testq{rret(), rret(), sf};
 732     v << jcci{CC_NZ, sf, next, rh.reenterTC};
 733     v = next;
 734     v << jmpi{rh.resumeHelper};
 735   });
 736 }
 737
 738 void emitInterpOneCFHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us,
 739                             const ResumeHelperEntryPoints& rh,
 740                             const CodeCache& code, Debug::DebugInfo& dbg) {
 741   alignJmpTarget(cb);
 742
 743   auto const emit = [&] (Op op, const char* name) {
 744     auto const stub = emitInterpOneCFHelper(cb, data, op, rh);
 745     us.interpOneCFHelpers[op] = stub;
 746     us.add(name, stub, code, dbg);
 747   };
 748
 749 #define O(name, imm, in, out, flags)          \
 750   if (((flags) & CF) || ((flags) & TF)) {     \
 751     emit(Op::name, "interpOneCFHelper"#name); \
 752   }
 753   OPCODES
 754 #undef O
 755
 756   // Exit is a very special snowflake.  Because it can appear in PHP
 757   // expressions, the emitter pretends that it pushed a value on the eval stack
 758   // (and iopExit actually does push Null right before throwing).  Marking it
 759   // as TF would mess up any bytecodes that want to consume its output value,
 760   // so we can't do that.  But we also don't want to extend regions past it, so
 761   // the JIT treats it as terminal and uses InterpOneCF to execute it.
 762   emit(Op::Exit, "interpOneCFHelperExit");
 763 }
 764
 765 ///////////////////////////////////////////////////////////////////////////////
 766
 767 TCA emitDecRefGeneric(CodeBlock& cb, DataBlock& data) {
 768   CGMeta meta;
 769
 770   auto const start = vwrap(cb, data, meta, [] (Vout& v) {
 771     v << vregrestrict{};
 772     v << stublogue{};
 773
 774     auto const rdata = rarg(0);
 775     auto const rtype = rarg(1);
 776
 777     auto const destroy = [&] (Vout& v) {
 778       // decRefGeneric is called via callfaststub, whose ABI claims that all
 779       // registers are preserved.  This is true in the fast path, but in the
 780       // slow path we need to manually save caller-saved registers.
 781       auto const callerSaved = abi().gpUnreserved - abi().calleeSaved;
 782       PhysRegSaver prs{v, callerSaved};
 783
 784       // As a consequence of being called via callfaststub, we can't safely use
 785       // any Vregs here except for status flags registers, at least not with
 786       // the default vwrap() ABI.  Just use the argument registers instead.
 787       assertx(callerSaved.contains(rdata));
 788       assertx(callerSaved.contains(rtype));
 789
 790       auto const dtor = lookupDestructor(v, rtype);
 791       v << callm{dtor, arg_regs(1)};
 792
 793       // The stub frame's saved RIP is at %rsp[8] before we saved the
 794       // caller-saved registers.
 795       v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
 796     };
 797
 798     emitDecRefWork(v, v, rdata, destroy, false);
 799
 800     v << stubret{};
 801   });
 802
 803   meta.process(nullptr);
 804   return start;
 805 }
 806
 807 ///////////////////////////////////////////////////////////////////////////////
 808
 809 TCA emitEnterTCExit(CodeBlock& cb, DataBlock& data, UniqueStubs& /*us*/) {
 810   return vwrap(cb, data, [&] (Vout& v) {
 811     // Eagerly save VM regs.
 812     storeVMRegs(v);
 813
 814     // Realign the native stack.
 815     switch (arch()) {
 816       case Arch::X64:
 817       case Arch::PPC64:
 818         v << lea{rsp()[8], rsp()};
 819         break;
 820       case Arch::ARM:
 821         break;
 822     }
 823
 824     // Store the return value on the top of the eval stack.  Whenever we get to
 825     // enterTCExit, we're semantically executing some PHP construct that sends
 826     // a return value out of a function (either a RetC, or a Yield, or an Await
 827     // that's suspending, etc), and moreover, we must be executing the return
 828     // that leaves this level of VM reentry (i.e. the only way we get here is
 829     // by coming from the callToExit stub or by a phpret{} or leavetc{} that
 830     // undoes the calltc{} or resumetc{} in enterTCHelper).
 831     //
 832     // Either way, we have a live PHP return value in the return registers,
 833     // which we need to put on the top of the evaluation stack.
 834     storeReturnRegs(v);
 835
 836     // Perform a native return.
 837     //
 838     // On PPC64, as there is no new frame created when entering the VM, the FP
 839     // must not be saved.
 840     v << stubret{RegSet(), arch() != Arch::PPC64};
 841   });
 842 }
 843
 844 TCA emitEnterTCHelper(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 845   alignJmpTarget(cb);
 846
 847   auto const sp       = rarg(0);
 848   auto const fp       = rarg(1);
 849   auto const start    = rarg(2);
 850   auto const firstAR  = rarg(3);
 851 #ifdef _MSC_VER
 852   auto const tl       = reg::r10;
 853   auto const calleeAR = reg::r11;
 854 #else
 855   auto const tl       = rarg(4);
 856   auto const calleeAR = rarg(5);
 857 #endif
 858
 859   return vwrap2(cb, cb, data, [&] (Vout& v, Vout& vc) {
 860     // Architecture-specific setup for entering the TC.
 861     v << inittc{};
 862
 863     // Native func prologue.
 864     v << stublogue{arch() != Arch::PPC64};
 865
 866 #ifdef _MSC_VER
 867     // Windows hates argument registers.
 868     v << load{rsp()[0x28], reg::r10};
 869     v << load{rsp()[0x30], reg::r11};
 870 #endif
 871
 872     // Set up linkage with the top VM frame in this nesting.
 873     v << store{rsp(), firstAR[AROFF(m_sfp)]};
 874
 875     // Set up the VM registers.
 876     v << copy{fp, rvmfp()};
 877     v << copy{sp, rvmsp()};
 878     v << copy{tl, rvmtl()};
 879
 880     // Unalign the native stack.
 881     switch (arch()) {
 882       case Arch::X64:
 883       case Arch::PPC64:
 884         v << lea{rsp()[-8], rsp()};
 885         break;
 886       case Arch::ARM:
 887         break;
 888     }
 889
 890     // Check if `calleeAR' was set.
 891     auto const sf = v.makeReg();
 892     v << testq{calleeAR, calleeAR, sf};
 893
 894     // We mark this block as unlikely in order to coax the emitter into
 895     // ordering this block last.  This is an important optimization for x64;
 896     // without it, both the jcc for the branch and the jmp for the resumetc{}
 897     // will end up in the same 16-byte extent of code, which messes up the
 898     // branch predictor.
 899     unlikelyIfThen(v, vc, CC_Z, sf, [&] (Vout& v) {
 900       // No callee means we're resuming in the middle of a TC function.
 901       v << resumetc{start, us.enterTCExit, vm_regs_with_sp()};
 902     });
 903
 904     // We have a callee; set rvmfp() and call it.
 905     v << copy{calleeAR, rvmfp()};
 906     v << calltc{start, rvmfp(), us.enterTCExit, vm_regs_with_sp()};
 907   });
 908 }
 909
 910 TCA emitHandleSRHelper(CodeBlock& cb, DataBlock& data) {
 911   alignJmpTarget(cb);
 912
 913   return vwrap(cb, data, [] (Vout& v) {
 914     storeVMRegs(v);
 915
 916     // Pack the service request args into a svcreq::ReqInfo on the stack.
 917     assertx(!(svcreq::kMaxArgs & 1));
 918     for (auto i = svcreq::kMaxArgs; i >= 2; i -= 2) {
 919       v << pushp{r_svcreq_arg(i - 1), r_svcreq_arg(i - 2)};
 920     }
 921     v << pushp{r_svcreq_stub(), r_svcreq_req()};
 922
 923     // Call mcg->handleServiceRequest(rsp()).
 924     auto const sp = v.makeReg();
 925     v << copy{rsp(), sp};
 926
 927     auto const ret = v.makeReg();
 928
 929     v << vcall{
 930       CallSpec::direct(svcreq::handleServiceRequest),
 931       v.makeVcallArgs({{sp}}),
 932       v.makeTuple({ret}),
 933       Fixup{},
 934       DestType::SSA
 935     };
 936
 937     // Pop the ReqInfo off the stack.
 938     auto const reqinfo_sz = static_cast<int>(sizeof(svcreq::ReqInfo));
 939     v << lea{rsp()[reqinfo_sz], rsp()};
 940
 941     // rvmtl() was preserved by the callee, but rvmsp() and rvmfp() might've
 942     // changed if we interpreted anything.  Reload them.  Also load the return
 943     // regs; if we're not returning, it's a spurious load.
 944     loadVMRegs(v);
 945     loadReturnRegs(v);
 946
 947     v << jmpr{ret};
 948   });
 949 }
 950
 951 ///////////////////////////////////////////////////////////////////////////////
 952
 953 TCA emitEndCatchHelper(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
 954   auto const udrspo = rvmtl()[unwinderDebuggerReturnSPOff()];
 955
 956   auto const debuggerReturn = vwrap(cb, data, [&] (Vout& v) {
 957     v << load{udrspo, rvmsp()};
 958     v << storeqi{0, udrspo};
 959   });
 960   svcreq::emit_persistent(cb, data, folly::none, REQ_POST_DEBUGGER_RET);
 961
 962   CGMeta meta;
 963
 964   auto const resumeCPPUnwind = vwrap(cb, data, meta, [&] (Vout& v) {
 965     static_assert(sizeof(tl_regState) == 8,
 966                   "The following store must match the size of tl_regState.");
 967     auto const regstate = emitTLSAddr(v, tls_datum(tl_regState));
 968     v << storeqi{static_cast<int32_t>(VMRegState::CLEAN), regstate};
 969
 970     v << load{rvmtl()[unwinderExnOff()], rarg(0)};
 971     v << call{TCA(_Unwind_Resume), arg_regs(1), &us.endCatchHelperPast};
 972     v << ud2{};
 973   });
 974   meta.process(nullptr);
 975
 976   alignJmpTarget(cb);
 977
 978   return vwrap(cb, data, [&] (Vout& v) {
 979     auto const done1 = v.makeBlock();
 980     auto const sf1 = v.makeReg();
 981
 982     v << cmpqim{0, udrspo, sf1};
 983     v << jcci{CC_NE, sf1, done1, debuggerReturn};
 984     v = done1;
 985
 986     // Normal end catch situation: call back to tc_unwind_resume, which returns
 987     // the catch trace (or null) in the first return register, and the new vmfp
 988     // in the second.
 989     v << copy{rvmfp(), rarg(0)};
 990     v << call{TCA(tc_unwind_resume), arg_regs(1)};
 991     v << copy{rret(1), rvmfp()};
 992
 993     auto const done2 = v.makeBlock();
 994     auto const sf2 = v.makeReg();
 995
 996     v << testq{rret(0), rret(0), sf2};
 997     v << jcci{CC_Z, sf2, done2, resumeCPPUnwind};
 998     v = done2;
 999
1000     v << jmpr{rret(0)};
1001   });
1002 }
1003
1004 TCA emitUnknownExceptionHandler(CodeBlock& cb,
1005                                 DataBlock& data,
1006                                 UniqueStubs& us) {
1007   alignJmpTarget(cb);
1008
1009   CGMeta meta;
1010   auto const ret = vwrap(cb, data, meta, [&] (Vout& v) {
1011     v << call{
1012       TCA(unknownExceptionHandler), {}, &us.unknownExceptionHandlerPast
1013     };
1014   });
1015   meta.process(nullptr);
1016
1017   return ret;
1018 }
1019
1020 TCA emitThrowSwitchMode(CodeBlock& cb, DataBlock& data) {
1021   alignJmpTarget(cb);
1022
1023   return vwrap(cb, data, [] (Vout& v) {
1024     v << call{TCA(throwSwitchMode)};
1025     v << ud2{};
1026   });
1027 }
1028
1029 template<class F>
1030 TCA emitHelperThunk(CodeCache& code, CodeBlock& cb, DataBlock& data, F* func) {
1031   // we only emit these calls into hot, main and cold.
1032   if (deltaFits(code.base() - (TCA)func, sz::dword) &&
1033       deltaFits(code.frozen().base() - (TCA)func, sz::dword)) {
1034     return (TCA)func;
1035   }
1036   alignJmpTarget(cb);
1037   return vwrap(cb, data, [&] (Vout& v) {
1038       v << jmpi{(TCA)func};
1039   });
1040 }
1041
1042 ///////////////////////////////////////////////////////////////////////////////
1043
1044 }
1045
1046 void UniqueStubs::emitAll(CodeCache& code, Debug::DebugInfo& dbg) {
1047   auto view = code.view();
1048   auto& main = view.main();
1049   auto& cold = view.cold();
1050   auto& frozen = view.frozen();
1051   auto& hotBlock = code.view(TransKind::Optimize).main();
1052   auto& data = view.data();
1053
1054   auto const hot = [&]() -> CodeBlock& {
1055     return hotBlock.available() > 512 ? hotBlock : main;
1056   };
1057
1058 #define ADD(name, stub) name = add(#name, (stub), code, dbg)
1059   ADD(enterTCExit,   emitEnterTCExit(main, data, *this));
1060   enterTCHelper =
1061     decltype(enterTCHelper)(add("enterTCHelper",
1062                                 emitEnterTCHelper(main, data, *this),
1063                                 code,
1064                                 dbg));
1065
1066   // These guys are required by a number of other stubs.
1067   ADD(handleSRHelper, emitHandleSRHelper(hot(), data));
1068   ADD(endCatchHelper, emitEndCatchHelper(frozen, data, *this));
1069   ADD(unknownExceptionHandler, emitUnknownExceptionHandler(cold, data, *this));
1070
1071   ADD(funcPrologueRedispatch, emitFuncPrologueRedispatch(hot(), data));
1072   ADD(fcallHelperThunk,       emitFCallHelperThunk(cold, frozen, data));
1073   ADD(funcBodyHelperThunk,    emitFuncBodyHelperThunk(cold, data));
1074   ADD(functionEnterHelper, emitFunctionEnterHelper(cold, frozen, data, *this));
1075   ADD(functionSurprisedOrStackOverflow,
1076       emitFunctionSurprisedOrStackOverflow(cold, frozen, data, *this));
1077
1078   ADD(retHelper,                  emitInterpRet(hot(), data));
1079   ADD(genRetHelper,               emitInterpGenRet<false>(cold, data));
1080   ADD(asyncGenRetHelper,          emitInterpGenRet<true>(hot(), data));
1081   ADD(retInlHelper,               emitInterpRet(hot(), data));
1082   ADD(debuggerRetHelper,          emitDebuggerInterpRet(cold, data));
1083   ADD(debuggerGenRetHelper,       emitDebuggerInterpGenRet<false>(cold, data));
1084   ADD(debuggerAsyncGenRetHelper,  emitDebuggerInterpGenRet<true>(cold, data));
1085
1086   ADD(bindCallStub,           emitBindCallStub<false>(cold, data));
1087   ADD(immutableBindCallStub,  emitBindCallStub<true>(cold, data));
1088   ADD(fcallArrayHelper,       emitFCallArrayHelper(hot(), frozen, data, *this));
1089
1090   ADD(decRefGeneric,  emitDecRefGeneric(cold, data));
1091
1092   ADD(callToExit,         emitCallToExit(main, data, *this));
1093   ADD(throwSwitchMode,    emitThrowSwitchMode(frozen, data));
1094
1095   ADD(handlePrimeCacheInit,
1096       emitHelperThunk(code, cold, data,
1097                       MethodCache::handlePrimeCacheInit<false>));
1098   ADD(handlePrimeCacheInitFatal,
1099       emitHelperThunk(code, cold, data,
1100                       MethodCache::handlePrimeCacheInit<true>));
1101   ADD(handleSlowPath,
1102       emitHelperThunk(code, main, data,
1103                       MethodCache::handleSlowPath<false>));
1104   ADD(handleSlowPathFatal,
1105       emitHelperThunk(code, main, data,
1106                       MethodCache::handleSlowPath<true>));
1107
1108 #undef ADD
1109
1110   add("freeLocalsHelpers",
1111       emitFreeLocalsHelpers(hot(), data, *this), code, dbg);
1112
1113   ResumeHelperEntryPoints rh;
1114   add("resumeInterpHelpers",
1115       emitResumeInterpHelpers(hot(), data, *this, rh),
1116       code, dbg);
1117   emitInterpOneCFHelpers(cold, data, *this, rh, code, dbg);
1118
1119   emitAllResumable(code, dbg);
1120 }
1121
1122 ///////////////////////////////////////////////////////////////////////////////
1123
1124 TCA UniqueStubs::add(const char* name, TCA start,
1125                      const CodeCache& code, Debug::DebugInfo& dbg) {
1126   if (!code.isValidCodeAddress(start)) return start;
1127
1128   auto& cb = code.blockFor(start);
1129   auto const end = cb.frontier();
1130
1131   FTRACE(1, "unique stub: {} @ {} -- {:4} bytes: {}\n",
1132          cb.name(),
1133          static_cast<void*>(start),
1134          static_cast<size_t>(end - start),
1135          name);
1136
1137   ONTRACE(2,
1138           [&]{
1139             std::ostringstream os;
1140             disasmRange(os, start, end);
1141             FTRACE(2, "{}\n", os.str());
1142           }()
1143          );
1144
1145   if (!RuntimeOption::EvalJitNoGdb) {
1146     dbg.recordStub(Debug::TCRange(start, end, &cb == &code.cold()),
1147                    folly::sformat("HHVM::{}", name));
1148   }
1149   if (RuntimeOption::EvalJitUseVtuneAPI) {
1150     reportHelperToVtune(folly::sformat("HHVM::{}", name).c_str(),
1151                         start,
1152                         end);
1153   }
1154   if (RuntimeOption::EvalPerfPidMap) {
1155     dbg.recordPerfMap(Debug::TCRange(start, end, &cb == &code.cold()),
1156                       SrcKey{},
1157                       nullptr,
1158                       false,
1159                       false,
1160                       folly::sformat("HHVM::{}", name));
1161   }
1162
1163   auto const newStub = StubRange{name, start, end};
1164   auto lower = std::lower_bound(m_ranges.begin(), m_ranges.end(), newStub);
1165
1166   // We assume ranges are non-overlapping.
1167   assertx(lower == m_ranges.end() || newStub.end <= lower->start);
1168   assertx(lower == m_ranges.begin() || (lower - 1)->end <= newStub.start);
1169   m_ranges.insert(lower, newStub);
1170   return start;
1171 }
1172
1173 std::string UniqueStubs::describe(TCA address) const {
1174   auto raw = [address] { return folly::sformat("{}", address); };
1175   if (m_ranges.empty()) return raw();
1176
1177   auto const dummy = StubRange{"", address, nullptr};
1178   auto lower = std::upper_bound(m_ranges.begin(), m_ranges.end(), dummy);
1179   if (lower == m_ranges.begin()) return raw();
1180
1181   --lower;
1182   if (lower->contains(address)) {
1183     return folly::sformat("{}+{:#x}", lower->name, address - lower->start);
1184   }
1185   return raw();
1186 }
1187
1188 ///////////////////////////////////////////////////////////////////////////////
1189
1190 RegSet interp_one_cf_regs() {
1191   return vm_regs_with_sp() | rarg(2);
1192 }
1193
1194 void emitInterpReq(Vout& v, SrcKey sk, FPInvOffset spOff) {
1195   if (sk.resumeMode() == ResumeMode::None) {
1196     v << lea{rvmfp()[-cellsToBytes(spOff.offset)], rvmsp()};
1197   }
1198   v << copy{v.cns(sk.pc()), rarg(0)};
1199   v << jmpi{tc::ustubs().interpHelper, arg_regs(1)};
1200 }
1201
1202 ///////////////////////////////////////////////////////////////////////////////
1203
1204 void enterTCImpl(TCA start, ActRec* stashedAR) {
1205   // We have to force C++ to spill anything that might be in a callee-saved
1206   // register (aside from rvmfp()), since enterTCHelper does not save them.
1207   CALLEE_SAVED_BARRIER();
1208   auto& regs = vmRegsUnsafe();
1209   tc::ustubs().enterTCHelper(regs.stack.top(), regs.fp, start,
1210                               vmFirstAR(), rds::tl_base, stashedAR);
1211   CALLEE_SAVED_BARRIER();
1212 }
1213
1214 ///////////////////////////////////////////////////////////////////////////////
1215
1216 }}