hphp/runtime/vm/jit/code-gen-x64.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/code-gen-x64.h"
  18
  19 #include <cstring>
  20 #include <unwind.h>
  21 #include <limits>
  22 #include <vector>
  23
  24 #include "folly/ScopeGuard.h"
  25 #include "folly/Format.h"
  26 #include "hphp/util/trace.h"
  27 #include "hphp/util/text-util.h"
  28 #include "hphp/util/abi-cxx.h"
  29
  30 #include "hphp/runtime/base/mixed-array.h"
  31 #include "hphp/runtime/base/comparisons.h"
  32 #include "hphp/runtime/base/complex-types.h"
  33 #include "hphp/runtime/base/runtime-option.h"
  34 #include "hphp/runtime/base/string-data.h"
  35 #include "hphp/runtime/base/types.h"
  36 #include "hphp/runtime/ext/ext_closure.h"
  37 #include "hphp/runtime/ext/ext_generator.h"
  38 #include "hphp/runtime/ext/ext_collections.h"
  39 #include "hphp/runtime/ext/asio/asio_blockable.h"
  40 #include "hphp/runtime/ext/asio/wait_handle.h"
  41 #include "hphp/runtime/ext/asio/async_function_wait_handle.h"
  42 #include "hphp/runtime/vm/bytecode.h"
  43 #include "hphp/runtime/vm/runtime.h"
  44 #include "hphp/runtime/base/stats.h"
  45 #include "hphp/runtime/base/rds.h"
  46 #include "hphp/runtime/base/rds-util.h"
  47 #include "hphp/runtime/vm/jit/arg-group.h"
  48 #include "hphp/runtime/vm/jit/back-end-x64.h"
  49 #include "hphp/runtime/vm/jit/cfg.h"
  50 #include "hphp/runtime/vm/jit/code-gen-helpers-x64.h"
  51 #include "hphp/runtime/vm/jit/ir.h"
  52 #include "hphp/runtime/vm/jit/layout.h"
  53 #include "hphp/runtime/vm/jit/mc-generator-internal.h"
  54 #include "hphp/runtime/vm/jit/mc-generator.h"
  55 #include "hphp/runtime/vm/jit/native-calls.h"
  56 #include "hphp/runtime/vm/jit/print.h"
  57 #include "hphp/runtime/vm/jit/prof-data.h"
  58 #include "hphp/runtime/vm/jit/reg-algorithms.h"
  59 #include "hphp/runtime/vm/jit/service-requests-inline.h"
  60 #include "hphp/runtime/vm/jit/service-requests-x64.h"
  61 #include "hphp/runtime/vm/jit/simplifier.h"
  62 #include "hphp/runtime/vm/jit/target-cache.h"
  63 #include "hphp/runtime/vm/jit/target-profile.h"
  64 #include "hphp/runtime/vm/jit/timer.h"
  65 #include "hphp/runtime/vm/jit/translator-inline.h"
  66 #include "hphp/runtime/vm/jit/translator.h"
  67 #include "hphp/runtime/vm/jit/types.h"
  68 #include "hphp/runtime/vm/jit/vasm-x64.h"
  69
  70 using HPHP::JIT::TCA;
  71
  72 namespace HPHP { namespace JIT { namespace X64 {
  73
  74 TRACE_SET_MOD(hhir);
  75
  76 namespace {
  77
  78 //////////////////////////////////////////////////////////////////////
  79
  80 using namespace JIT::reg;
  81
  82 /*
  83  * It's not normally ok to directly use tracelet abi registers in
  84  * codegen, unless you're directly dealing with an instruction that
  85  * does near-end-of-tracelet glue.  (Or also we sometimes use them
  86  * just for some static_assertions relating to calls to helpers from
  87  * mcg that hardcode these registers.)
  88  */
  89
  90 void cgPunt(const char* file, int line, const char* func, uint32_t bcOff,
  91             const Func* vmFunc, bool resumed, TransID profTransId) {
  92   if (dumpIREnabled()) {
  93     HPHP::Trace::trace("--------- CG_PUNT %s %d %s  bcOff: %d \n",
  94                        file, line, func, bcOff);
  95   }
  96   throw FailedCodeGen(file, line, func, bcOff, vmFunc, resumed, profTransId);
  97 }
  98
  99 #define CG_PUNT(instr)                                            \
 100   cgPunt(__FILE__, __LINE__, #instr, m_curInst->marker().bcOff(), \
 101          curFunc(), resumed(), m_curInst->marker().profTransId())
 102
 103 const char* getContextName(const Class* ctx) {
 104   return ctx ? ctx->name()->data() : ":anonymous:";
 105 }
 106
 107 } // unnamed namespace
 108 //////////////////////////////////////////////////////////////////////
 109
 110 template <class Block>
 111 void CodeGenerator::ifBlock(ConditionCode cc, Block taken,
 112                             bool unlikely) {
 113   if (unlikely) return unlikelyIfBlock(cc, taken);
 114   ifThen(m_as, cc, taken);
 115 }
 116
 117 template <class Then>
 118 void CodeGenerator::unlikelyIfBlock(Vout& v, Vout& vcold,
 119                                     ConditionCode cc, Then then) {
 120   auto unlikely = vcold.makeBlock();
 121   auto done = v.makeBlock();
 122   v << jcc{cc, {done, unlikely}};
 123   vcold = unlikely;
 124   then(vcold);
 125   if (!vcold.closed()) vcold << jmp{done};
 126   v = done;
 127 }
 128
 129 template <class Block>
 130 void CodeGenerator::ifBlock(Vout& v, Vout& vcold, ConditionCode cc,
 131                             Block taken, bool unlikely) {
 132   if (unlikely) return unlikelyIfBlock(v, vcold, cc, taken);
 133   auto takenLabel = v.makeBlock();
 134   auto doneLabel = v.makeBlock();
 135   v << jcc{cc, {doneLabel, takenLabel}};
 136   v = takenLabel;
 137   taken(v);
 138   if (!v.closed()) v << jmp{doneLabel};
 139   v = doneLabel;
 140 }
 141
 142 template <class Block>
 143 void CodeGenerator::unlikelyIfBlock(ConditionCode cc, Block unlikely) {
 144   if (m_as.base() == m_acold.base()) {
 145     ifThen(m_as, cc, unlikely);
 146   } else {
 147     Label unlikelyLabel, done;
 148     m_as.jcc(cc, unlikelyLabel);
 149     asm_label(m_acold, unlikelyLabel);
 150     unlikely(m_acold);
 151     m_acold.jmp(done);
 152     asm_label(m_as, done);
 153   }
 154 }
 155
 156 // Generate an if-then-else block
 157 template <class Then, class Else>
 158 void CodeGenerator::ifThenElse(Asm& a, ConditionCode cc, Then thenBlock,
 159                                Else elseBlock) {
 160   Label elseLabel, done;
 161   a.jcc8(ccNegate(cc), elseLabel);
 162   thenBlock(a);
 163   a.jmp8(done);
 164   asm_label(a, elseLabel);
 165   elseBlock(a);
 166   asm_label(a, done);
 167 }
 168
 169 template <class Then, class Else>
 170 void CodeGenerator::ifThenElse(ConditionCode cc, Then thenBlock,
 171                                Else elseBlock, bool unlikely) {
 172   if (unlikely) return unlikelyIfThenElse(cc, thenBlock, elseBlock);
 173   ifThenElse(m_as, cc, thenBlock, elseBlock);
 174 }
 175
 176 template <class Then, class Else>
 177 void CodeGenerator::ifThenElse(Vout& v, ConditionCode cc,
 178                                Then thenBlock, Else elseBlock) {
 179   auto thenLabel = v.makeBlock();
 180   auto elseLabel = v.makeBlock();
 181   auto done = v.makeBlock();
 182   v << jcc{cc, {elseLabel, thenLabel}};
 183   v = thenLabel;
 184   thenBlock(v);
 185   if (!v.closed()) v << jmp{done};
 186   v = elseLabel;
 187   elseBlock(v);
 188   if (!v.closed()) v << jmp{done};
 189   v = done;
 190 }
 191
 192 template <class Then, class Else>
 193 void CodeGenerator::unlikelyIfThenElse(ConditionCode cc, Then unlikely,
 194                                        Else elseBlock) {
 195   if (m_as.base() == m_acold.base()) {
 196     Label elseLabel, done;
 197     m_as.jcc8(ccNegate(cc), elseLabel);
 198     unlikely(m_as);
 199     m_as.jmp8(done);
 200     asm_label(m_as, elseLabel);
 201     elseBlock(m_as);
 202     asm_label(m_as, done);
 203   } else {
 204     Label unlikelyLabel, done;
 205     m_as.jcc(cc, unlikelyLabel);
 206     elseBlock(m_as);
 207     asm_label(m_acold, unlikelyLabel);
 208     unlikely(m_acold);
 209     m_acold.jmp(done);
 210     asm_label(m_as, done);
 211   }
 212 }
 213
 214 template <class Then, class Else>
 215 void CodeGenerator::ifThenElse(Vout& v, Vout& vcold, ConditionCode cc,
 216                                Then thenBlock, Else elseBlock, bool unlikely) {
 217   if (unlikely) return unlikelyIfThenElse(v, vcold, cc, thenBlock, elseBlock);
 218   ifThenElse(v, cc, thenBlock, elseBlock);
 219 }
 220
 221 template <class Then, class Else>
 222 void CodeGenerator::unlikelyIfThenElse(Vout& v, Vout& vcold, ConditionCode cc,
 223                                        Then unlikelyBlock, Else elseBlock) {
 224   auto elseLabel = v.makeBlock();
 225   auto unlikelyLabel = vcold.makeBlock();
 226   auto done = v.makeBlock();
 227   v << jcc{cc, {elseLabel, unlikelyLabel}};
 228   v = elseLabel;
 229   elseBlock(v);
 230   if (!v.closed()) v << jmp{done};
 231   vcold = unlikelyLabel;
 232   unlikelyBlock(vcold);
 233   if (!vcold.closed()) vcold << jmp{done};
 234   v = done;
 235 }
 236
 237 /*
 238  * Select a scratch register to use in the given instruction, prefering the
 239  * lower registers which don't require a REX prefix.  The selected register
 240  * must not be any of the instructions inputs or outputs, and neither a register
 241  * that is alive across this instruction.
 242  */
 243 PhysReg CodeGenerator::selectScratchReg(IRInstruction* inst) {
 244   // TODO: #3634984 this function must die.
 245   static const RegSet kLowGPRegs = RegSet()
 246     | RegSet(reg::rax)
 247     | RegSet(reg::rcx)
 248     | RegSet(reg::rdx)
 249     | RegSet(reg::rsi)
 250     | RegSet(reg::rdi)
 251     ;
 252   RegSet liveRegs = m_state.liveRegs[inst];
 253   liveRegs |= m_state.regs.srcRegs(*inst);
 254   liveRegs |= m_state.regs.dstRegs(*inst);
 255   PhysReg selectedReg;
 256   if ((kLowGPRegs - liveRegs).findFirst(selectedReg)) {
 257     return selectedReg;
 258   }
 259   return rCgGP;
 260 }
 261
 262 void CodeGenerator::cgInst(IRInstruction* inst) {
 263   Opcode opc = inst->op();
 264   m_curInst = inst;
 265   m_instRegs = &m_state.regs[inst];
 266   m_rScratch = selectScratchReg(inst);
 267   SCOPE_EXIT { m_curInst = nullptr; };
 268   {
 269     Vauto vasm(&m_state.meta);
 270     m_vmain = &vasm.main(m_mainCode);
 271     m_vcold = &vasm.cold(m_coldCode);
 272     m_vfrozen = &vasm.frozen(m_frozenCode);
 273     SCOPE_EXIT { m_vmain = m_vcold = m_vfrozen = nullptr; };
 274     switch (opc) {
 275 #define O(name, dsts, srcs, flags)                                \
 276       case name: FTRACE(7, "cg" #name "\n");                          \
 277              cg ## name (inst);                                   \
 278              return;
 279       IR_OPCODES
 280 #undef O
 281
 282       default:
 283         always_assert(false);
 284     }
 285   }
 286 }
 287
 288 #define NOOP_OPCODE(opcode) \
 289   void CodeGenerator::cg##opcode(IRInstruction*) {}
 290
 291 #define CALL_OPCODE(opcode) \
 292   void CodeGenerator::cg##opcode(IRInstruction* i) { cgCallNative(vmain(), i); }
 293
 294 #define CALL_STK_OPCODE(opcode) \
 295   CALL_OPCODE(opcode)           \
 296   CALL_OPCODE(opcode ## Stk)
 297
 298 NOOP_OPCODE(DefConst)
 299 NOOP_OPCODE(DefFP)
 300 NOOP_OPCODE(TrackLoc)
 301 NOOP_OPCODE(AssertLoc)
 302 NOOP_OPCODE(AssertStk)
 303 NOOP_OPCODE(Nop)
 304 NOOP_OPCODE(DefLabel)
 305 NOOP_OPCODE(ExceptionBarrier)
 306 NOOP_OPCODE(TakeStack)
 307 NOOP_OPCODE(TakeRef)
 308 NOOP_OPCODE(EndGuards)
 309
 310 CALL_OPCODE(AddElemStrKey)
 311 CALL_OPCODE(AddElemIntKey)
 312 CALL_OPCODE(AddNewElem)
 313 CALL_OPCODE(ArrayAdd)
 314 CALL_OPCODE(Box)
 315 CALL_OPCODE(ColAddElemC)
 316 CALL_OPCODE(ColAddNewElemC)
 317
 318 CALL_OPCODE(ConvBoolToArr);
 319 CALL_OPCODE(ConvDblToArr);
 320 CALL_OPCODE(ConvIntToArr);
 321 CALL_OPCODE(ConvObjToArr);
 322 CALL_OPCODE(ConvStrToArr);
 323 CALL_OPCODE(ConvCellToArr);
 324
 325 CALL_OPCODE(ConvStrToBool);
 326 CALL_OPCODE(ConvCellToBool);
 327
 328 CALL_OPCODE(ConvArrToDbl);
 329 CALL_OPCODE(ConvObjToDbl);
 330 CALL_OPCODE(ConvStrToDbl);
 331 CALL_OPCODE(ConvCellToDbl);
 332
 333 CALL_OPCODE(ConvArrToInt);
 334 CALL_OPCODE(ConvObjToInt);
 335 CALL_OPCODE(ConvStrToInt);
 336 CALL_OPCODE(ConvCellToInt);
 337
 338 CALL_OPCODE(ConvCellToObj);
 339
 340 CALL_OPCODE(ConvDblToStr);
 341 CALL_OPCODE(ConvIntToStr);
 342 CALL_OPCODE(ConvObjToStr);
 343 CALL_OPCODE(ConvResToStr);
 344 CALL_OPCODE(ConvCellToStr);
 345
 346 CALL_OPCODE(ConcatStrStr);
 347 CALL_OPCODE(ConcatStrInt);
 348 CALL_OPCODE(ConcatIntStr);
 349 CALL_OPCODE(ConcatStr3);
 350 CALL_OPCODE(ConcatStr4);
 351
 352 CALL_OPCODE(TypeProfileFunc)
 353 CALL_OPCODE(CreateCont)
 354 CALL_OPCODE(CreateAFWH)
 355 CALL_OPCODE(CreateSSWH)
 356 CALL_OPCODE(AFWHPrepareChild)
 357 CALL_OPCODE(ABCUnblock)
 358 CALL_OPCODE(NewArray)
 359 CALL_OPCODE(NewMixedArray)
 360 CALL_OPCODE(NewMIArray)
 361 CALL_OPCODE(NewMSArray)
 362 CALL_OPCODE(NewLikeArray)
 363 CALL_OPCODE(NewPackedArray)
 364 CALL_OPCODE(NewCol)
 365 CALL_OPCODE(Clone)
 366 CALL_OPCODE(AllocObj)
 367 CALL_OPCODE(CustomInstanceInit)
 368 CALL_OPCODE(InitProps)
 369 CALL_OPCODE(InitSProps)
 370 CALL_OPCODE(LdClsCtor)
 371 CALL_OPCODE(LookupClsMethod)
 372 CALL_OPCODE(LookupClsRDSHandle)
 373 CALL_OPCODE(LdArrFuncCtx)
 374 CALL_OPCODE(LdArrFPushCuf)
 375 CALL_OPCODE(LdStrFPushCuf)
 376 CALL_OPCODE(PrintStr)
 377 CALL_OPCODE(PrintInt)
 378 CALL_OPCODE(PrintBool)
 379 CALL_OPCODE(DbgAssertPtr)
 380 CALL_OPCODE(LdSwitchDblIndex)
 381 CALL_OPCODE(LdSwitchStrIndex)
 382 CALL_OPCODE(LdSwitchObjIndex)
 383 CALL_OPCODE(VerifyParamCallable)
 384 CALL_OPCODE(VerifyParamFail)
 385 CALL_OPCODE(VerifyRetCallable)
 386 CALL_OPCODE(VerifyRetFail)
 387 CALL_OPCODE(RaiseUninitLoc)
 388 CALL_OPCODE(WarnNonObjProp)
 389 CALL_OPCODE(ThrowNonObjProp)
 390 CALL_OPCODE(RaiseUndefProp)
 391 CALL_OPCODE(RaiseError)
 392 CALL_OPCODE(RaiseWarning)
 393 CALL_OPCODE(RaiseNotice)
 394 CALL_OPCODE(RaiseArrayIndexNotice)
 395 CALL_OPCODE(IncStatGrouped)
 396 CALL_OPCODE(ClosureStaticLocInit)
 397 CALL_OPCODE(ArrayIdx)
 398 CALL_OPCODE(GenericIdx)
 399 CALL_OPCODE(LdClsPropAddrOrNull)
 400 CALL_OPCODE(LdClsPropAddrOrRaise)
 401 CALL_OPCODE(LdGblAddrDef)
 402
 403 // Vector instruction helpers
 404 CALL_OPCODE(BaseG)
 405 CALL_OPCODE(PropX)
 406 CALL_STK_OPCODE(PropDX)
 407 CALL_OPCODE(CGetProp)
 408 CALL_STK_OPCODE(VGetProp)
 409 CALL_STK_OPCODE(BindProp)
 410 CALL_STK_OPCODE(SetProp)
 411 CALL_OPCODE(UnsetProp)
 412 CALL_STK_OPCODE(SetOpProp)
 413 CALL_STK_OPCODE(IncDecProp)
 414 CALL_OPCODE(EmptyProp)
 415 CALL_OPCODE(IssetProp)
 416 CALL_OPCODE(ElemX)
 417 CALL_OPCODE(ElemArray)
 418 CALL_STK_OPCODE(ElemDX)
 419 CALL_STK_OPCODE(ElemUX)
 420 CALL_OPCODE(ArrayGet)
 421 CALL_OPCODE(StringGet)
 422 CALL_OPCODE(MapGet)
 423 CALL_OPCODE(CGetElem)
 424 CALL_STK_OPCODE(VGetElem)
 425 CALL_STK_OPCODE(BindElem)
 426 CALL_STK_OPCODE(SetWithRefElem)
 427 CALL_STK_OPCODE(SetWithRefNewElem)
 428 CALL_OPCODE(ArraySet)
 429 CALL_OPCODE(MapSet)
 430 CALL_OPCODE(ArraySetRef)
 431 CALL_STK_OPCODE(SetElem)
 432 CALL_STK_OPCODE(UnsetElem)
 433 CALL_STK_OPCODE(SetOpElem)
 434 CALL_STK_OPCODE(IncDecElem)
 435 CALL_STK_OPCODE(SetNewElem)
 436 CALL_STK_OPCODE(SetNewElemArray)
 437 CALL_STK_OPCODE(BindNewElem)
 438 CALL_OPCODE(ArrayIsset)
 439 CALL_OPCODE(VectorIsset)
 440 CALL_OPCODE(PairIsset)
 441 CALL_OPCODE(MapIsset)
 442 CALL_OPCODE(IssetElem)
 443 CALL_OPCODE(EmptyElem)
 444
 445 CALL_OPCODE(InstanceOfIface)
 446 CALL_OPCODE(InterfaceSupportsArr)
 447 CALL_OPCODE(InterfaceSupportsStr)
 448 CALL_OPCODE(InterfaceSupportsInt)
 449 CALL_OPCODE(InterfaceSupportsDbl)
 450
 451 CALL_OPCODE(ZeroErrorLevel)
 452 CALL_OPCODE(RestoreErrorLevel)
 453
 454 CALL_OPCODE(Count)
 455
 456 CALL_OPCODE(SurpriseHook)
 457 CALL_OPCODE(FunctionSuspendHook)
 458 CALL_OPCODE(FunctionReturnHook)
 459
 460 CALL_OPCODE(OODeclExists)
 461
 462 #undef NOOP_OPCODE
 463
 464 // Thread chain of patch locations using the 4 byte space in each jmp/jcc
 465 static void prependPatchAddr(CodegenState& state,
 466                              Block* block,
 467                              TCA patchAddr) {
 468   auto &patches = state.patches;
 469   ssize_t diff = patches[block] ? (patchAddr - (TCA)patches[block]) : 0;
 470   assert(deltaFits(diff, sz::dword));
 471   *(int32_t*)(patchAddr) = (int32_t)diff;
 472   patches[block] = patchAddr;
 473 }
 474
 475 static void emitFwdJmp(Asm& a, Block* target, CodegenState& state) {
 476   if (auto addr = state.addresses[target]) {
 477     return a.jmpAuto(addr);
 478   }
 479
 480   // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
 481   a.jmp(a.frontier());
 482   TCA immPtr = a.frontier() - 4;
 483   prependPatchAddr(state, target, immPtr);
 484 }
 485
 486 void emitFwdJmp(CodeBlock& cb, Block* target, CodegenState& state) {
 487   Asm a { cb };
 488   emitFwdJmp(a, target, state);
 489 }
 490
 491 void CodeGenerator::emitFwdJcc(Asm& a, ConditionCode cc, Block* target) {
 492   if (auto addr = m_state.addresses[target]) {
 493     return a.jccAuto(cc, addr);
 494   }
 495
 496   // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
 497   a.jcc(cc, a.frontier());
 498   TCA immPtr = a.frontier() - 4;
 499   prependPatchAddr(m_state, target, immPtr);
 500 }
 501
 502 void CodeGenerator::emitFwdJcc(ConditionCode cc, Block* target) {
 503   emitFwdJcc(m_as, cc, target);
 504 }
 505
 506 void CodeGenerator::emitLoadImm(Asm& as, int64_t val, PhysReg dstReg) {
 507   Vauto().main(as) << ldimm{val, dstReg};
 508 }
 509
 510 /*
 511  * Returns a XMM register containing the value of SSATmp tmp,
 512  * which can be either a bool, an int, or a double.
 513  * If the value is already in a XMM register, simply returns it.
 514  * Otherwise, the value is moved into rCgXMM, which is returned.
 515  * If instructions to convert to a double at runtime are needed,
 516  * they're emitted in 'as'.
 517  * TODO: #3634984, #3727837 This function must die.
 518  */
 519 PhysReg CodeGenerator::prepXMMReg(Vout& v, const SSATmp* src,
 520                                   const PhysLoc& srcLoc, RegXMM rtmp) {
 521   assert(src->isA(Type::Bool) || src->isA(Type::Int) || src->isA(Type::Dbl));
 522   always_assert(srcLoc.reg() != InvalidReg);
 523   auto rsrc = srcLoc.reg();
 524
 525   // Case 1: src is already in a XMM register
 526   if (rsrc.isSIMD()) {
 527     return rsrc;
 528   }
 529
 530   // Case 2: src Dbl stored in GP reg
 531   if (src->isA(Type::Dbl)) {
 532     v << copy{rsrc, rtmp};
 533     return rtmp;
 534   }
 535
 536   // Case 2.b: Bool or Int stored in GP reg
 537   zeroExtendIfBool(v, src, rsrc);
 538   v << cvtsi2sd{rsrc, rtmp};
 539   return rtmp;
 540 }
 541
 542 VregXMM CodeGenerator::prepXMM(Vout& v, const SSATmp* src,
 543                                const PhysLoc& srcLoc) {
 544   assert(src->isA(Type::Bool) || src->isA(Type::Int) || src->isA(Type::Dbl));
 545   always_assert(srcLoc.reg() != InvalidReg);
 546   auto rsrc = srcLoc.reg();
 547
 548   // Case 1: src is already in a XMM register
 549   if (rsrc.isSIMD()) {
 550     return rsrc;
 551   }
 552
 553   // Case 2: src Dbl stored in GP reg
 554   if (src->isA(Type::Dbl)) {
 555     auto rtmp = v.makeReg();
 556     v << copy{rsrc, rtmp};
 557     return rtmp;
 558   }
 559
 560   // Case 2.b: Bool or Int stored in GP reg
 561   zeroExtendIfBool(v, src, rsrc);
 562   auto rtmp = v.makeReg();
 563   v << cvtsi2sd{rsrc, rtmp};
 564   return rtmp;
 565 }
 566
 567 PhysReg CodeGenerator::prepXMMReg(Asm& a, const SSATmp* src,
 568                                   const PhysLoc& srcLoc, RegXMM rtmp) {
 569   return prepXMMReg(Vauto().main(a), src, srcLoc, rtmp);
 570 }
 571
 572 void CodeGenerator::emitCompare(IRInstruction* inst) {
 573   emitCompare(Vauto().main(m_mainCode), inst);
 574 }
 575
 576 void CodeGenerator::emitCompare(Vout& v, IRInstruction* inst) {
 577   auto src0 = inst->src(0);
 578   auto src1 = inst->src(1);
 579   auto loc0 = srcLoc(0);
 580   auto loc1 = srcLoc(1);
 581   auto const type0 = src0->type();
 582   auto const type1 = src1->type();
 583
 584   // can't generate CMP instructions correctly for anything that isn't
 585   // a bool or a numeric, and we can't mix bool/numerics because
 586   // -1 == true in PHP, but not in HHIR binary representation
 587   if (!((type0 <= Type::Int  && type1 <= Type::Int) ||
 588         (type0 <= Type::Bool && type1 <= Type::Bool) ||
 589         (type0 <= Type::Cls  && type1 <= Type::Cls))) {
 590     CG_PUNT(emitCompare);
 591   }
 592   auto reg0 = loc0.reg();
 593   auto reg1 = loc1.reg();
 594
 595   if (reg1 == InvalidReg) {
 596     if (type0 <= Type::Bool) {
 597       v << cmpbi{src1->boolVal(), reg0};
 598     } else {
 599       v << cmpqi{safe_cast<int32_t>(src1->intVal()), reg0};
 600     }
 601   } else {
 602     // Note the reverse syntax in the assembler.
 603     // This cmp will compute reg0 - reg1
 604     if (type0 <= Type::Bool) {
 605       v << cmpb{reg1, reg0};
 606     } else {
 607       v << cmpq{reg1, reg0};
 608     }
 609   }
 610 }
 611
 612 void CodeGenerator::emitCompareInt(IRInstruction* inst) {
 613   emitCompareInt(Vauto().main(m_mainCode), inst);
 614 }
 615
 616 void CodeGenerator::emitCompareInt(Vout& v, IRInstruction* inst) {
 617   auto srcReg0 = srcLoc(0).reg();
 618   auto srcReg1 = srcLoc(1).reg();
 619   if (srcReg1 == InvalidReg) {
 620     v << cmpqi{safe_cast<int32_t>(inst->src(1)->intVal()), srcReg0};
 621   } else {
 622     // Note the reverse syntax in the assembler.
 623     // This cmp will compute srcReg0 - srcReg1
 624     v << cmpq{srcReg1, srcReg0};
 625   }
 626 }
 627
 628 void CodeGenerator::emitReqBindJcc(Vout& v, ConditionCode cc,
 629                                    const ReqBindJccData* extra) {
 630   v << bindjcc1{cc, {extra->notTaken, extra->taken}};
 631 }
 632
 633 void CodeGenerator::cgDefSP(IRInstruction* inst) {
 634   if (RuntimeOption::EvalHHIRGenerateAsserts && !inst->marker().resumed()) {
 635     // Verify that rbx == rbp - spOff
 636     m_as.lea(rbp[-cellsToBytes(inst->extra<StackOffset>()->offset)],
 637              m_rScratch);
 638     m_as.cmpq(m_rScratch, rbx);
 639     ifBlock(CC_NE, [](Asm& a) { a.ud2(); });
 640   }
 641 }
 642
 643 void CodeGenerator::cgCheckNullptr(IRInstruction* inst) {
 644   if (!inst->taken()) return;
 645   auto reg = srcLoc(0).reg(0);
 646   m_as.testq (reg, reg);
 647   emitFwdJcc(CC_NZ, inst->taken());
 648 }
 649
 650 void CodeGenerator::cgCheckNonNull(IRInstruction* inst) {
 651   auto srcReg = srcLoc(0).reg();
 652   auto dstReg = dstLoc(0).reg();
 653   auto taken  = inst->taken();
 654   assert(taken);
 655
 656   m_as.testq (srcReg, srcReg);
 657   emitFwdJcc(CC_Z, taken);
 658   if (dstReg != InvalidReg) emitMovRegReg(m_as, srcReg, dstReg);
 659 }
 660
 661 void CodeGenerator::cgAssertNonNull(IRInstruction* inst) {
 662   auto& v = vmain();
 663   auto srcReg = srcLoc(0).reg();
 664   auto dstReg = dstLoc(0).reg();
 665   if (RuntimeOption::EvalHHIRGenerateAsserts) {
 666     v << testq{srcReg, srcReg};
 667     ifThen(v, CC_Z, [&](Vout& v) {
 668       v << ud2{};
 669     });
 670   }
 671   v << copy{srcReg, dstReg};
 672 }
 673
 674 void CodeGenerator::cgAssertType(IRInstruction* inst) {
 675   auto const srcRegs = srcLoc(0);
 676   auto const dstRegs = dstLoc(0);
 677   shuffle2(vmain(), srcRegs.reg(0), srcRegs.reg(1),
 678            dstRegs.reg(0), dstRegs.reg(1));
 679 }
 680
 681 void CodeGenerator::cgLdUnwinderValue(IRInstruction* inst) {
 682   cgLoad(inst->dst(), dstLoc(0), rVmTl[unwinderTvOff()], inst->taken());
 683 }
 684
 685 void CodeGenerator::cgBeginCatch(IRInstruction* inst) {
 686   auto const& info = m_state.catches[inst->block()];
 687   auto& v = vmain();
 688   auto catchStart = v.makePoint();
 689   v << point{catchStart};
 690   v << fixupcatch{Vpoint{size_t(info.afterCall)}, catchStart};
 691   v << incstat{Stats::TC_CatchTrace};
 692
 693   // We want to restore state as though the call had completed
 694   // successfully, so skip over any stack arguments and pop any
 695   // saved registers.
 696   if (info.rspOffset) {
 697     v << addqi{info.rspOffset, rsp, rsp};
 698   }
 699   PhysRegSaverParity::emitPops(v, info.savedRegs);
 700 }
 701
 702 static void unwindResumeHelper(_Unwind_Exception* data) {
 703   tl_regState = VMRegState::CLEAN;
 704   _Unwind_Resume(data);
 705 }
 706
 707 static void callUnwindResumeHelper(Vout& v) {
 708   v << loadq{rVmTl[unwinderScratchOff()], rdi};
 709   v << call{(TCA)unwindResumeHelper}; // pass control back to the unwinder
 710   v << ud2{};
 711 }
 712
 713 void CodeGenerator::cgEndCatch(IRInstruction* inst) {
 714   callUnwindResumeHelper(vmain());
 715 }
 716
 717 void CodeGenerator::cgTryEndCatch(IRInstruction* inst) {
 718   auto& v = vmain();
 719   v << cmpbim{0, rVmTl[unwinderSideExitOff()]};
 720   unlikelyIfBlock(v, vcold(), CC_E, callUnwindResumeHelper);
 721
 722   // doSideExit == true, so fall through to the side exit code
 723   v << incstat{Stats::TC_CatchSideExit};
 724 }
 725
 726 void CodeGenerator::cgDeleteUnwinderException(IRInstruction* inst) {
 727   auto& v = vmain();
 728   v << loadq{rVmTl[unwinderScratchOff()], rdi};
 729   v << call{(TCA)_Unwind_DeleteException};
 730 }
 731
 732 void CodeGenerator::cgJcc(IRInstruction* inst) {
 733   emitCompare(inst);
 734   emitFwdJcc(opToConditionCode(inst->op()), inst->taken());
 735 }
 736
 737 void CodeGenerator::cgJccInt(IRInstruction* inst) {
 738   emitCompareInt(inst);
 739   emitFwdJcc(opToConditionCode(inst->op()), inst->taken());
 740 }
 741
 742 void CodeGenerator::cgReqBindJcc(IRInstruction* inst) {
 743   // TODO(#2404427): prepareForTestAndSmash?
 744   auto& v = vmain();
 745   emitCompare(v, inst);
 746   emitReqBindJcc(v, opToConditionCode(inst->op()),
 747                  inst->extra<ReqBindJccData>());
 748 }
 749
 750 void CodeGenerator::cgReqBindJccInt(IRInstruction* inst) {
 751   // TODO(#2404427): prepareForTestAndSmash?
 752   auto& v = vmain();
 753   emitCompareInt(v, inst);
 754   emitReqBindJcc(v, opToConditionCode(inst->op()),
 755                  inst->extra<ReqBindJccData>());
 756 }
 757
 758 void CodeGenerator::cgJmpGt(IRInstruction* i)    { cgJcc(i); }
 759 void CodeGenerator::cgJmpGte(IRInstruction* i)   { cgJcc(i); }
 760 void CodeGenerator::cgJmpLt(IRInstruction* i)    { cgJcc(i); }
 761 void CodeGenerator::cgJmpLte(IRInstruction* i)   { cgJcc(i); }
 762 void CodeGenerator::cgJmpEq(IRInstruction* i)    { cgJcc(i); }
 763 void CodeGenerator::cgJmpNeq(IRInstruction* i)   { cgJcc(i); }
 764 void CodeGenerator::cgJmpSame(IRInstruction* i)  { cgJcc(i); }
 765 void CodeGenerator::cgJmpNSame(IRInstruction* i) { cgJcc(i); }
 766
 767 void CodeGenerator::cgReqBindJmpGt(IRInstruction* i)    { cgReqBindJcc(i); }
 768 void CodeGenerator::cgReqBindJmpGte(IRInstruction* i)   { cgReqBindJcc(i); }
 769 void CodeGenerator::cgReqBindJmpLt(IRInstruction* i)    { cgReqBindJcc(i); }
 770 void CodeGenerator::cgReqBindJmpLte(IRInstruction* i)   { cgReqBindJcc(i); }
 771 void CodeGenerator::cgReqBindJmpEq(IRInstruction* i)    { cgReqBindJcc(i); }
 772 void CodeGenerator::cgReqBindJmpNeq(IRInstruction* i)   { cgReqBindJcc(i); }
 773 void CodeGenerator::cgReqBindJmpSame(IRInstruction* i)  { cgReqBindJcc(i); }
 774 void CodeGenerator::cgReqBindJmpNSame(IRInstruction* i) { cgReqBindJcc(i); }
 775
 776 void CodeGenerator::cgSideExitJmpGt(IRInstruction* i)    { cgExitJcc(i); }
 777 void CodeGenerator::cgSideExitJmpGte(IRInstruction* i)   { cgExitJcc(i); }
 778 void CodeGenerator::cgSideExitJmpLt(IRInstruction* i)    { cgExitJcc(i); }
 779 void CodeGenerator::cgSideExitJmpLte(IRInstruction* i)   { cgExitJcc(i); }
 780 void CodeGenerator::cgSideExitJmpEq(IRInstruction* i)    { cgExitJcc(i); }
 781 void CodeGenerator::cgSideExitJmpNeq(IRInstruction* i)   { cgExitJcc(i); }
 782 void CodeGenerator::cgSideExitJmpSame(IRInstruction* i)  { cgExitJcc(i); }
 783 void CodeGenerator::cgSideExitJmpNSame(IRInstruction* i) { cgExitJcc(i); }
 784
 785 void CodeGenerator::cgJmpGtInt(IRInstruction* i)    { cgJccInt(i); }
 786 void CodeGenerator::cgJmpGteInt(IRInstruction* i)   { cgJccInt(i); }
 787 void CodeGenerator::cgJmpLtInt(IRInstruction* i)    { cgJccInt(i); }
 788 void CodeGenerator::cgJmpLteInt(IRInstruction* i)   { cgJccInt(i); }
 789 void CodeGenerator::cgJmpEqInt(IRInstruction* i)    { cgJccInt(i); }
 790 void CodeGenerator::cgJmpNeqInt(IRInstruction* i)   { cgJccInt(i); }
 791
 792 void CodeGenerator::cgReqBindJmpGtInt(IRInstruction* i)  { cgReqBindJccInt(i); }
 793 void CodeGenerator::cgReqBindJmpGteInt(IRInstruction* i) { cgReqBindJccInt(i); }
 794 void CodeGenerator::cgReqBindJmpLtInt(IRInstruction* i)  { cgReqBindJccInt(i); }
 795 void CodeGenerator::cgReqBindJmpLteInt(IRInstruction* i) { cgReqBindJccInt(i); }
 796 void CodeGenerator::cgReqBindJmpEqInt(IRInstruction* i)  { cgReqBindJccInt(i); }
 797 void CodeGenerator::cgReqBindJmpNeqInt(IRInstruction* i) { cgReqBindJccInt(i); }
 798
 799 void CodeGenerator::cgSideExitJmpGtInt(IRInstruction* i)  { cgExitJccInt(i); }
 800 void CodeGenerator::cgSideExitJmpGteInt(IRInstruction* i) { cgExitJccInt(i); }
 801 void CodeGenerator::cgSideExitJmpLtInt(IRInstruction* i)  { cgExitJccInt(i); }
 802 void CodeGenerator::cgSideExitJmpLteInt(IRInstruction* i) { cgExitJccInt(i); }
 803 void CodeGenerator::cgSideExitJmpEqInt(IRInstruction* i)  { cgExitJccInt(i); }
 804 void CodeGenerator::cgSideExitJmpNeqInt(IRInstruction* i) { cgExitJccInt(i); }
 805
 806 //////////////////////////////////////////////////////////////////////
 807
 808 void CodeGenerator::cgHalt(IRInstruction* inst) {
 809   m_as.ud2();
 810 }
 811
 812 //////////////////////////////////////////////////////////////////////
 813
 814 /**
 815  * Once the arg sources and dests are all assigned; emit moves and exchanges to
 816  * put all the args in desired registers. Any arguments that don't fit in
 817  * registers will be put on the stack. In addition to moves and exchanges,
 818  * shuffleArgs also handles adding lea-offsets for dest registers (dest = src +
 819  * lea-offset) and zero extending bools (dest = zeroExtend(src)).
 820  */
 821 static bool shuffleArgsPlanningHelper(PhysReg::Map<PhysReg>& moves,
 822                                       PhysReg::Map<ArgDesc*>& argDescs,
 823                                       ArgDesc& arg) {
 824   auto kind = arg.kind();
 825   if (!(kind == ArgDesc::Kind::Reg  ||
 826         kind == ArgDesc::Kind::Addr ||
 827         kind == ArgDesc::Kind::TypeReg)) {
 828     return true;
 829   }
 830   auto dstReg = arg.dstReg();
 831   auto srcReg = arg.srcReg();
 832   if (dstReg != srcReg) {
 833     moves[dstReg] = srcReg;
 834     argDescs[dstReg] = &arg;
 835   }
 836   return false;
 837 }
 838
 839 static int64_t shuffleArgs(Vout& v, ArgGroup& args, CppCall& call) {
 840   // Compute the move/shuffle plan.
 841   PhysReg::Map<PhysReg> moves;
 842   PhysReg::Map<ArgDesc*> argDescs;
 843
 844   for (size_t i = 0; i < args.numGpArgs(); ++i) {
 845     auto& arg = args.gpArg(i);
 846     if (shuffleArgsPlanningHelper(moves, argDescs, arg)) {
 847       continue;
 848     }
 849     switch (call.kind()) {
 850     case CppCall::Kind::Indirect:
 851       if (arg.dstReg() == call.reg()) {
 852         // an indirect call uses an argument register for the func ptr.
 853         // Use rax instead and update the CppCall
 854         moves[reg::rax] = call.reg();
 855         call.updateCallIndirect(reg::rax);
 856       }
 857       break;
 858     case CppCall::Kind::Direct:
 859     case CppCall::Kind::Virtual:
 860     case CppCall::Kind::ArrayVirt:
 861       break;
 862     }
 863   }
 864   for (size_t i = 0; i < args.numSimdArgs(); ++i) {
 865     shuffleArgsPlanningHelper(moves, argDescs, args.simdArg(i));
 866   }
 867
 868   // Store any arguments past the initial 6 to the stack. This has to happen
 869   // before the shuffles below in case the shuffles would clobber any of the
 870   // srcRegs here.
 871   for (int i = args.numStackArgs() - 1; i >= 0; --i) {
 872     auto& arg = args.stkArg(i);
 873     auto srcReg = arg.srcReg();
 874     assert(arg.dstReg() == InvalidReg);
 875     switch (arg.kind()) {
 876       case ArgDesc::Kind::Reg:
 877         if (arg.isZeroExtend()) {
 878           auto tmp = v.makeReg();
 879           v << movzbl{srcReg, tmp};
 880           v << push{tmp};
 881         } else {
 882           if (srcReg.isSIMD()) {
 883             auto tmp = v.makeReg();
 884             v << copy{srcReg, tmp};
 885             v << push{tmp};
 886           } else {
 887             v << push{srcReg};
 888           }
 889         }
 890         break;
 891
 892       case ArgDesc::Kind::TypeReg:
 893         static_assert(kTypeWordOffset == 0 || kTypeWordOffset == 1,
 894                       "kTypeWordOffset value not supported");
 895         assert(srcReg.isGP());
 896         // x86 stacks grow down, so push higher offset items first
 897         if (kTypeWordOffset == 0) {
 898           v << push{srcReg};
 899         } else {
 900           // 4 bytes of garbage:
 901           v << pushl{eax};
 902           // get the type in the right place in rTmp before pushing it
 903           auto tmp1 = v.makeReg();
 904           v << movb{srcReg, tmp1};
 905           auto tmp2 = v.makeReg();
 906           v << shlli{CHAR_BIT, tmp1, tmp2};
 907           v << pushl{tmp2};
 908         }
 909         break;
 910
 911       case ArgDesc::Kind::Imm: {
 912         auto tmp = v.makeReg();
 913         v << ldimm{arg.imm(), tmp};
 914         v << push{tmp};
 915         break;
 916       }
 917
 918       case ArgDesc::Kind::Addr: {
 919         auto tmp = v.makeReg();
 920         v << lea{arg.srcReg()[arg.disp().l()], tmp};
 921         v << push{tmp};
 922         break;
 923       }
 924
 925       case ArgDesc::Kind::IpRel: {
 926         auto tmp = v.makeReg();
 927         v << leap{rip[arg.imm().q()], tmp};
 928         v << push{tmp};
 929         break;
 930       }
 931
 932       case ArgDesc::Kind::None:
 933         v << push{rax};
 934         if (RuntimeOption::EvalHHIRGenerateAsserts) {
 935           emitImmStoreq(v, 0xbadbadbadbadbad, *rsp);
 936         }
 937         break;
 938     }
 939   }
 940
 941   // Execute the plan
 942   int num_moves = 0;
 943   auto rTmp = rAsm;
 944   auto const howTo = doRegMoves(moves, rTmp);
 945   for (auto& how : howTo) {
 946     switch (how.m_kind) {
 947       case MoveInfo::Kind::Move: {
 948         num_moves++;
 949         if (how.m_dst == rTmp) {
 950           v << copy{how.m_src, how.m_dst};
 951         } else {
 952           ArgDesc* argDesc = argDescs[how.m_dst];
 953           if (argDesc == nullptr) {
 954             // when no ArgDesc is available is a straight reg to reg copy
 955             v << copy{how.m_src, how.m_dst};
 956           } else {
 957             ArgDesc::Kind kind = argDesc->kind();
 958             if (kind == ArgDesc::Kind::Reg || kind == ArgDesc::Kind::TypeReg) {
 959               if (argDesc->isZeroExtend()) {
 960                 assert(how.m_src.isGP());
 961                 assert(how.m_dst.isGP());
 962                 v << movzbl{how.m_src, how.m_dst};
 963               } else {
 964                 v << copy{how.m_src, how.m_dst};
 965               }
 966             } else {
 967               assert(kind == ArgDesc::Kind::Addr);
 968               assert(how.m_src.isGP());
 969               assert(how.m_dst.isGP());
 970               v << lea{how.m_src[argDesc->disp().l()], how.m_dst};
 971             }
 972             if (kind != ArgDesc::Kind::TypeReg) {
 973               argDesc->markDone();
 974             }
 975           }
 976         }
 977         break;
 978       }
 979     case MoveInfo::Kind::Xchg:
 980       num_moves++;
 981       assert(how.m_src.isGP());
 982       assert(how.m_dst.isGP());
 983       v << copy2{how.m_src, how.m_dst, how.m_dst, how.m_src};
 984       break;
 985     }
 986   }
 987   if (num_moves > 0) {
 988     TRACE(2, "arg-moves %d\n", num_moves);
 989   }
 990
 991   // Handle const-to-register moves, type shifting,
 992   // load-effective address and zero extending for bools.
 993   // Ignore args that have been handled by the
 994   // move above.
 995   for (size_t i = 0; i < args.numGpArgs(); ++i) {
 996     auto& arg = args.gpArg(i);
 997     if (arg.done()) continue;
 998     ArgDesc::Kind kind = arg.kind();
 999     PhysReg dst = arg.dstReg();
1000     assert(dst.isGP());
1001     if (kind == ArgDesc::Kind::Imm) {
1002       v << ldimm{arg.imm().q(), dst};
1003     } else if (kind == ArgDesc::Kind::TypeReg) {
1004       if (kTypeShiftBits > 0) {
1005         v << shlqi{kTypeShiftBits, dst, dst};
1006       }
1007     } else if (kind == ArgDesc::Kind::Addr) {
1008       v << addqi{arg.disp(), dst, dst};
1009     } else if (kind == ArgDesc::Kind::IpRel) {
1010       v << leap{rip[arg.imm().q()], dst};
1011     } else if (arg.isZeroExtend()) {
1012       v << movzbl{dst, dst};
1013     } else if (RuntimeOption::EvalHHIRGenerateAsserts &&
1014                kind == ArgDesc::Kind::None) {
1015       v << ldimm{0xbadbadbadbadbad, dst};
1016     }
1017   }
1018
1019   for (size_t i = 0; i < args.numSimdArgs(); ++i) {
1020     auto &arg = args.simdArg(i);
1021     if (arg.done()) continue;
1022     auto kind = arg.kind();
1023     auto dst = arg.dstReg();
1024     assert(dst.isSIMD());
1025     if (kind == ArgDesc::Kind::Imm) {
1026       v << ldimm{arg.imm().q(), dst};
1027     } else if (RuntimeOption::EvalHHIRGenerateAsserts &&
1028                kind == ArgDesc::Kind::None) {
1029       v << ldimm{0xbadbadbadbadbad, dst};
1030     }
1031   }
1032
1033   return args.numStackArgs() * sizeof(int64_t);
1034 }
1035
1036 void CodeGenerator::cgCallNative(Vout& v, IRInstruction* inst) {
1037   using namespace NativeCalls;
1038
1039   Opcode opc = inst->op();
1040   always_assert(CallMap::hasInfo(opc));
1041
1042   auto const& info = CallMap::info(opc);
1043   ArgGroup argGroup = info.toArgGroup(m_state.regs, inst);
1044
1045   auto call = [&]() -> CppCall {
1046     switch (info.func.type) {
1047     case FuncType::Call:
1048       return CppCall(info.func.call);
1049     case FuncType::SSA:
1050       return CppCall::direct(
1051         reinterpret_cast<void (*)()>(inst->src(info.func.srcIdx)->tcaVal()));
1052     }
1053     not_reached();
1054   }();
1055
1056   auto const dest = [&]() -> CallDest {
1057     switch (info.dest) {
1058       case DestType::None:  return kVoidDest;
1059       case DestType::TV:    return callDestTV(inst);
1060       case DestType::SSA:   return callDest(inst);
1061       case DestType::Dbl:   return callDestDbl(inst);
1062     }
1063     not_reached();
1064   }();
1065
1066   cgCallHelper(v, call, dest, info.sync, argGroup);
1067 }
1068
1069 CallDest CodeGenerator::callDest(PhysReg reg0,
1070                                  PhysReg reg1 /* = InvalidReg */) const {
1071   return { DestType::SSA, reg0, reg1 };
1072 }
1073
1074 CallDest CodeGenerator::callDest(const IRInstruction* inst) const {
1075   if (!inst->numDsts()) return kVoidDest;
1076   auto loc = dstLoc(0);
1077   return { DestType::SSA, loc.reg(0), loc.reg(1) };
1078 }
1079
1080 CallDest CodeGenerator::callDestTV(const IRInstruction* inst) const {
1081   if (!inst->numDsts()) return kVoidDest;
1082   auto loc = dstLoc(0);
1083   return { DestType::TV, loc.reg(0), loc.reg(1) };
1084 }
1085
1086 CallDest CodeGenerator::callDestDbl(const IRInstruction* inst) const {
1087   if (!inst->numDsts()) return kVoidDest;
1088   auto loc = dstLoc(0);
1089   return { DestType::Dbl, loc.reg(0), loc.reg(1) };
1090 }
1091
1092 void
1093 CodeGenerator::cgCallHelper(Asm& a, CppCall call, const CallDest& dstInfo,
1094                             SyncOptions sync, ArgGroup& args) {
1095   cgCallHelper(a, call, dstInfo, sync, args, m_state.liveRegs[m_curInst]);
1096 }
1097
1098 void
1099 CodeGenerator::cgCallHelper(Vout& v, CppCall call, const CallDest& dstInfo,
1100                             SyncOptions sync, ArgGroup& args) {
1101   return cgCallHelper(v, call, dstInfo, sync, args,
1102                       m_state.liveRegs[m_curInst]);
1103 }
1104
1105 void
1106 CodeGenerator::cgCallHelper(Asm& a, CppCall call, const CallDest& dstInfo,
1107                             SyncOptions sync, ArgGroup& args, RegSet toSave) {
1108   cgCallHelper(Vauto(&m_state.meta).main(a), call, dstInfo, sync, args,
1109                toSave);
1110 }
1111
1112 void
1113 CodeGenerator::cgCallHelper(Vout& v, CppCall call, const CallDest& dstInfo,
1114                             SyncOptions sync, ArgGroup& args, RegSet toSave) {
1115   assert(m_curInst->isNative());
1116
1117   auto const destType = dstInfo.type;
1118   auto const dstReg0  = dstInfo.reg0;
1119   auto const dstReg1  = dstInfo.reg1;
1120
1121   // Save the caller-saved registers that are live across this
1122   // instruction. The number of regs to save and the number of args
1123   // being passed on the stack affect the parity of the PhysRegSaver,
1124   // so we use the generic version here.
1125   toSave = toSave & kCallerSaved;
1126   assert((toSave & RegSet().add(dstReg0).add(dstReg1)).empty());
1127   assert(IMPLIES(m_curInst->is(Call, CallArray), toSave.empty()));
1128   PhysRegSaverParity regSaver(1 + args.numStackArgs(), v, toSave);
1129
1130   // Assign registers to the arguments then prepare them for the call.
1131   for (size_t i = 0; i < args.numGpArgs(); i++) {
1132     args.gpArg(i).setDstReg(argNumToRegName[i]);
1133   }
1134   for (size_t i = 0; i < args.numSimdArgs(); i++) {
1135     args.simdArg(i).setDstReg(argNumToSIMDRegName[i]);
1136   }
1137   regSaver.bytesPushed(shuffleArgs(v, args, call));
1138
1139   // do the call; may use a trampoline
1140   if (sync == SyncOptions::kSmashableAndSyncPoint) {
1141     assert(call.kind() == CppCall::Kind::Direct);
1142     v << mccall{(TCA)call.address()};
1143   } else {
1144     emitCall(v, call);
1145   }
1146   auto afterCall = v.makePoint();
1147   v << point{afterCall};
1148   if (RuntimeOption::HHProfServerEnabled || sync != SyncOptions::kNoSyncPoint) {
1149     // if we are profiling the heap, we always need to sync because
1150     // regs need to be correct during smart allocations no matter
1151     // what
1152     recordSyncPoint(v, sync);
1153   }
1154
1155   auto* taken = m_curInst->taken();
1156   if (taken && taken->isCatch()) {
1157     always_assert_flog(
1158       sync != SyncOptions::kNoSyncPoint,
1159       "cgCallHelper called with kNoSyncPoint but inst has a catch block: {}\n",
1160       *m_curInst
1161     );
1162     always_assert_flog(
1163       taken->catchMarker() == m_curInst->marker(),
1164       "Catch trace doesn't match fixup:\n"
1165       "Instruction: {}\n"
1166       "Catch trace: {}\n"
1167       "Fixup      : {}\n",
1168       m_curInst->toString(),
1169       taken->catchMarker().show(),
1170       m_curInst->marker().show()
1171     );
1172
1173     auto& info = m_state.catches[taken];
1174     info.afterCall = (TCA)size_t(afterCall);
1175     info.savedRegs = toSave;
1176     info.rspOffset = regSaver.rspAdjustment();
1177   } else if (!m_curInst->is(Call, CallArray, ContEnter)) {
1178     // The current instruction doesn't have a catch block so it'd better not
1179     // throw. Register a null catch trace to indicate this to the
1180     // unwinder. Call and CallArray don't have catch blocks because they smash
1181     // all live values and optimizations are aware of this.
1182     v << fixupcatch{afterCall, v.makePoint()};
1183   }
1184
1185   // copy the call result to the destination register(s)
1186   switch (destType) {
1187   case DestType::TV:
1188     {
1189       // rax contains m_type and m_aux but we're expecting just the
1190       // type in the lower bits, so shift the type result register.
1191       auto rval = packed_tv ? reg::rdx : reg::rax;
1192       auto rtyp = packed_tv ? reg::rax : reg::rdx;
1193       if (kTypeShiftBits > 0) v << shrqi{kTypeShiftBits, rtyp, rtyp};
1194       shuffle2(v, rval, rtyp, dstReg0, dstReg1);
1195     }
1196     break;
1197   case DestType::SSA:
1198     // copy the single-register result to dstReg0
1199     assert(dstReg1 == InvalidReg);
1200     if (dstReg0 != InvalidReg) v << copy{reg::rax, dstReg0};
1201     break;
1202   case DestType::None:
1203     // void return type, no registers have values
1204     assert(dstReg0 == InvalidReg && dstReg1 == InvalidReg);
1205     break;
1206   case DestType::Dbl:
1207     // copy the single-register result to dstReg0
1208     assert(dstReg1 == InvalidReg);
1209     if (dstReg0 != InvalidReg) v << copy{reg::xmm0, dstReg0};
1210     break;
1211   }
1212 }
1213
1214 void CodeGenerator::cgMov(IRInstruction* inst) {
1215   always_assert(inst->src(0)->numWords() == inst->dst(0)->numWords());
1216   Vauto vasm;
1217   auto& v = vasm.main(m_as);
1218   if (srcLoc(0).hasReg(1)) {
1219     assert(srcLoc(0).reg() != InvalidReg);
1220     shuffle2(v, srcLoc(0).reg(), srcLoc(0).reg(1),
1221              dstLoc(0).reg(), dstLoc(0).reg(1));
1222     return;
1223   }
1224   auto const src = inst->src(0);
1225
1226   auto sreg = srcLoc(0).reg();
1227   auto dreg = dstLoc(0).reg();
1228   if (sreg != InvalidReg && dreg != InvalidReg) {
1229     v << copy{sreg, dreg};
1230   } else if (sreg == InvalidReg && dreg != InvalidReg) {
1231     // It won't have a raw value if it's null.
1232     auto const stype = src->type();
1233     assert(stype.hasRawVal() || stype <= Type::Null);
1234     auto const raw = stype.hasRawVal() ? src->rawVal() : 0;
1235     v << ldimm{raw, dreg};
1236   } else {
1237     assert(sreg == InvalidReg && dreg == InvalidReg);
1238   }
1239 }
1240
1241 template<class OpInstr>
1242 void CodeGenerator::cgUnaryIntOp(PhysLoc dst_loc,
1243                                  SSATmp* src, PhysLoc src_loc) {
1244   assert(src->isA(Type::Int));
1245   assert(dst_loc.reg() != InvalidReg);
1246   assert(src_loc.reg() != InvalidReg);
1247   auto dstReg = dst_loc.reg();
1248   auto srcReg = src_loc.reg();
1249   auto& v = vmain();
1250
1251   // Integer operations require 64-bit representations
1252   zeroExtendIfBool(v, src, srcReg);
1253   v << copy{srcReg, dstReg};
1254   v << OpInstr{dstReg, dstReg};
1255 }
1256
1257 template<class OpInstr>
1258 void cgUnaryIntOp(Asm& a, PhysLoc dst_loc, SSATmp* src, PhysLoc src_loc,
1259                   OpInstr instr) {
1260   assert(src->isA(Type::Int));
1261   assert(dst_loc.reg() != InvalidReg);
1262   assert(src_loc.reg() != InvalidReg);
1263   auto dstReg = dst_loc.reg();
1264   auto srcReg = src_loc.reg();
1265
1266   // Integer operations require 64-bit representations
1267   zeroExtendIfBool(a, src, srcReg);
1268
1269   emitMovRegReg(a, srcReg, dstReg);
1270   (a.*instr)   (dstReg);
1271 }
1272
1273 void CodeGenerator::cgAbsDbl(IRInstruction* inst) {
1274   auto srcReg = srcLoc(0).reg();
1275   auto dstReg = dstLoc(0).reg();
1276   auto& v = vmain();
1277
1278   // clear the high bit
1279   auto resReg = dstReg.isSIMD() ? Vreg(dstReg) : v.makeReg();
1280   Vreg tmp_src;
1281   if (srcReg.isSIMD()) {
1282     tmp_src = srcReg;
1283   } else {
1284     tmp_src = v.makeReg();
1285     v << copy{srcReg, tmp_src};
1286   }
1287   auto tmp1 = v.makeReg();
1288   v << psllq{1, tmp_src, tmp1};
1289   v << psrlq{1, tmp1, resReg};
1290   if (resReg != Vreg(dstReg)) v << copy{resReg, dstReg};
1291 }
1292
1293 typedef void (Asm::*AsmInstrIR)(Immed, Reg64);
1294
1295 void CodeGenerator::cgBinaryIntOp(IRInstruction* inst,
1296                                   void (Asm::*instrIR)(Immed, Reg64),
1297                                   void (Asm::*instrRR)(Reg64, Reg64),
1298                                   Commutativity commuteFlag) {
1299   assert(m_curInst == inst); // could remove the inst param
1300   const SSATmp* src0  = inst->src(0);
1301   const SSATmp* src1  = inst->src(1);
1302
1303   // inputs must be ints, or a (bool,bool) operation that ends up behaving
1304   // like an int anyway (e.g. XorBool)
1305   assert((src0->isA(Type::Int) && src1->isA(Type::Int)));
1306
1307   auto const dstReg      = dstLoc(0).reg();
1308   auto const src0Reg     = srcLoc(0).reg();
1309   auto const src1Reg     = srcLoc(1).reg();
1310   auto& a                = m_as;
1311
1312   auto opWithScratch = [&]() {
1313     a.movq(src0Reg, m_rScratch);
1314     (a.*instrRR) (src1Reg, m_rScratch);
1315     a.movq(m_rScratch, dstReg);
1316   };
1317
1318   // Two registers.
1319   if (src0Reg != InvalidReg && src1Reg != InvalidReg) {
1320     if (dstReg == src0Reg) {
1321       (a.*instrRR)  (src1Reg, dstReg);
1322     } else if (dstReg == src1Reg) {
1323       if (commuteFlag == Commutative) {
1324         (a.*instrRR) (src0Reg, dstReg);
1325       } else {
1326         opWithScratch();
1327       }
1328     } else {
1329       emitMovRegReg(a, src0Reg, dstReg);
1330       (a.*instrRR) (src1Reg, dstReg);
1331     }
1332     return;
1333   }
1334
1335   auto intVal = [] (const SSATmp* s) {
1336     return s->isA(Type::Int) ? safe_cast<int32_t>(s->intVal()) :
1337            int32_t(s->boolVal());
1338   };
1339
1340   // One register, and one immediate.
1341   if (commuteFlag == Commutative) {
1342     assert(instrIR);
1343     auto imm = intVal(src1Reg == InvalidReg ? src1 : src0);
1344     auto srcReg = srcLoc(src1Reg == InvalidReg ? 0 : 1).reg();
1345     if (srcReg == dstReg) {
1346       (a.*instrIR) (imm, dstReg);
1347     } else {
1348       a. emitImmReg(imm, dstReg);
1349       (a.*instrRR) (srcReg, dstReg);
1350     }
1351     return;
1352   }
1353
1354   // NonCommutative:
1355   if (src0Reg == InvalidReg) {
1356     if (dstReg == src1Reg) {
1357       emitLoadImm(a, intVal(src0), m_rScratch);
1358       (a.*instrRR) (src1Reg, m_rScratch);
1359       a.movq(m_rScratch, dstReg);
1360     } else {
1361       emitLoadImm(a, intVal(src0), dstReg);
1362       (a.*instrRR) (src1Reg, dstReg);
1363     }
1364     return;
1365   }
1366
1367   assert(src1Reg == InvalidReg);
1368   emitMovRegReg(a, src0Reg, dstReg);
1369   (a.*instrIR) (intVal(src1), dstReg);
1370 }
1371
1372 template<class Op, class Opi>
1373 void CodeGenerator::cgBinaryIntOp(IRInstruction* inst) {
1374   assert(m_curInst == inst); // could remove the inst param
1375   UNUSED const SSATmp* src0  = inst->src(0);
1376   UNUSED const SSATmp* src1  = inst->src(1);
1377
1378   // inputs must be ints, or a (bool,bool) operation that ends up behaving
1379   // like an int anyway (e.g. XorBool)
1380   assert((src0->isA(Type::Int) && src1->isA(Type::Int)) ||
1381          (src0->isA(Type::Bool) && src1->isA(Type::Bool)));
1382
1383   auto const dstReg      = dstLoc(0).reg();
1384   auto const src0Reg     = srcLoc(0).reg();
1385   auto const src1Reg     = srcLoc(1).reg();
1386   auto& v                = vmain();
1387
1388   // LHS must always be assigned a register.
1389   assert(src0Reg != InvalidReg);
1390
1391   if (src1Reg != InvalidReg) {
1392     // Two registers
1393     v << Op{src1Reg, src0Reg, dstReg};
1394   } else {
1395     // One register, one immediate
1396     auto imm = src1->isA(Type::Int) ? safe_cast<int32_t>(src1->intVal()) :
1397                int32_t(src1->boolVal());
1398     v << Opi{imm, src0Reg, dstReg};
1399   }
1400 }
1401
1402 template<class Emit>
1403 void CodeGenerator::cgBinaryDblOp(IRInstruction* inst, Emit emit) {
1404   assert(inst == m_curInst);
1405   const SSATmp* src0  = inst->src(0);
1406   const SSATmp* src1  = inst->src(1);
1407   auto loc0 = srcLoc(0);
1408   auto loc1 = srcLoc(1);
1409   assert(src0->isA(Type::Dbl) && src1->isA(Type::Dbl));
1410
1411   auto& v = vmain();
1412   auto dstReg = dstLoc(0).reg();
1413   auto resReg = dstReg.isSIMD() && dstReg != loc1.reg() ? Vreg(dstReg) :
1414                 v.makeReg();
1415   assert(resReg.isVirt() || resReg.isSIMD());
1416
1417   auto srcReg0 = prepXMM(v, src0, loc0);
1418   auto srcReg1 = prepXMM(v, src1, loc1);
1419
1420   emit(v, srcReg1, srcReg0, resReg);
1421   if (resReg != Vreg(dstReg)) v << copy{resReg, dstReg};
1422 }
1423
1424 /*
1425  * If src2 is 1, this generates dst = src1 - 1 or src1 + 1 using the inc
1426  * or dec x86 instructions. The return value is whether or not the
1427  * instruction could be generated.
1428  */
1429 bool emitIncDecHelper(Asm& as, PhysLoc dst, SSATmp* src1, PhysLoc loc1,
1430                       SSATmp* src2, PhysLoc loc2,
1431                       void(Asm::*emitFunc)(Reg64)) {
1432   if (loc1.reg() != InvalidReg && loc2.reg() != InvalidReg &&
1433       src2->isConst(1)) {
1434     emitMovRegReg(as, loc1.reg(), dst.reg());
1435     (as.*emitFunc)(dst.reg());
1436     return true;
1437   }
1438   return false;
1439 }
1440
1441 void CodeGenerator::cgAddIntO(IRInstruction* inst) {
1442   assert(inst->taken() != nullptr);
1443   SSATmp* src1 = inst->src(0);
1444   SSATmp* src2 = inst->src(1);
1445   auto loc1 = srcLoc(0);
1446   auto loc2 = srcLoc(1);
1447   auto dst = dstLoc(0);
1448
1449   // Special cases: x = y + 1, x = 1 + y
1450   if (!emitIncDecHelper(m_as, dst, src1, loc1, src2, loc2, &Asm::incq) &&
1451       !emitIncDecHelper(m_as, dst, src2, loc2, src1, loc1, &Asm::incq)) {
1452     cgBinaryIntOp(inst, &Asm::addq, &Asm::addq, Commutative);
1453   }
1454   emitFwdJcc(m_as, CC_O, inst->taken());
1455 }
1456
1457 // r1 = r0 - r1
1458 // r1 = -r1 + r0
1459
1460 void CodeGenerator::cgSubIntO(IRInstruction* inst) {
1461   assert(inst->taken() != nullptr);
1462   auto src1 = inst->src(0);
1463   auto src2 = inst->src(1);
1464   auto loc1 = srcLoc(0);
1465   auto loc2 = srcLoc(1);
1466   auto dst = dstLoc(0);
1467
1468   if (src1->isConst(0)) {
1469     // There is no unary negate HHIR instruction, so handle that here.
1470     X64::cgUnaryIntOp(m_as, dst, src2, loc2, &Asm::neg);
1471   } else if (!emitIncDecHelper(m_as, dst, src1, loc1, src2, loc2, &Asm::decq)) {
1472     cgBinaryIntOp(inst, &Asm::subq, &Asm::subq, NonCommutative);
1473   }
1474   emitFwdJcc(m_as, CC_O, inst->taken());
1475 }
1476
1477 void CodeGenerator::cgMulIntO(IRInstruction* inst) {
1478   assert(inst->taken() != nullptr);
1479   auto src0Reg = srcLoc(0).reg();
1480   auto src1Reg = srcLoc(1).reg();
1481   auto dstReg = dstLoc(0).reg();
1482   if (dstReg != src0Reg && dstReg == src1Reg) {
1483     m_as.imul(src0Reg, dstReg);
1484   } else {
1485     emitMovRegReg(m_as, src0Reg, dstReg);
1486     m_as.imul(src1Reg, dstReg);
1487   }
1488   emitFwdJcc(m_as, CC_O, inst->taken());
1489 }
1490
1491 /*
1492  * If src2 is 1, this generates dst = src1 - 1 or src1 + 1 using the inc
1493  * or dec x86 instructions. The return value is whether or not the
1494  * instruction could be generated.
1495  */
1496 template<class Inst>
1497 bool CodeGenerator::emitIncDec(PhysLoc dst, SSATmp* src1, PhysLoc loc1,
1498                                SSATmp* src2, PhysLoc loc2) {
1499   auto& v = vmain();
1500   if (loc1.reg() != InvalidReg && loc2.reg() != InvalidReg &&
1501       src2->isConst(1)) {
1502     v << copy{loc1.reg(), dst.reg()};
1503     v << Inst{dst.reg(), dst.reg()};
1504     return true;
1505   }
1506   return false;
1507 }
1508
1509 void CodeGenerator::cgRoundCommon(IRInstruction* inst, RoundDirection dir) {
1510   auto src = inst->src(0);
1511   auto dstReg = dstLoc(0).reg();
1512   auto& v = vmain();
1513   auto inReg  = prepXMM(v, src, srcLoc(0));
1514   auto outReg = dstReg.isSIMD() ? Vreg(dstReg) : v.makeReg();
1515   v << roundsd{dir, inReg, outReg};
1516   v << copy{outReg, dstReg};
1517 }
1518
1519 void CodeGenerator::cgFloor(IRInstruction* inst) {
1520   cgRoundCommon(inst, RoundDirection::floor);
1521 }
1522
1523 void CodeGenerator::cgCeil(IRInstruction* inst) {
1524   cgRoundCommon(inst, RoundDirection::ceil);
1525 }
1526
1527 void CodeGenerator::cgAddInt(IRInstruction* inst) {
1528   SSATmp* src0 = inst->src(0);
1529   SSATmp* src1 = inst->src(1);
1530   auto loc0 = srcLoc(0);
1531   auto loc1 = srcLoc(1);
1532   auto dst = dstLoc(0);
1533
1534   // Special cases: x = y + 1, x = 1 + y
1535   if (emitIncDec<incq>(dst, src0, loc0, src1, loc1) ||
1536       emitIncDec<incq>(dst, src1, loc1, src0, loc1)) {
1537     return;
1538   }
1539
1540   cgBinaryIntOp<addq,addqi>(inst);
1541 }
1542
1543 void CodeGenerator::cgSubInt(IRInstruction* inst) {
1544   auto src0 = inst->src(0);
1545   auto src1 = inst->src(1);
1546   auto loc0 = srcLoc(0);
1547   auto loc1 = srcLoc(1);
1548   auto dst = dstLoc(0);
1549
1550   if (emitIncDec<decq>(dst, src0, loc0, src1, loc1)) return;
1551
1552   if (src0->isConst(0)) {
1553     // There is no unary negate HHIR instruction, so handle that here.
1554     cgUnaryIntOp<neg>(dst, src1, loc1);
1555     return;
1556   }
1557
1558   // not using cgBinaryIntOp because sub is not commutative, and we can do
1559   // r1=r0-r1 by doing neg r1; addq r1+=r0 without a scratch register.
1560   auto s0 = loc0.reg();
1561   auto s1 = loc1.reg();
1562   auto d = dst.reg();
1563   assert(s0 != InvalidReg && d != InvalidReg);
1564   auto& v = vmain();
1565   if (s1 == d) {
1566     v << neg{s1, d};
1567     v << addq{s0, s1, d};
1568   } else if (s1 == InvalidReg) {
1569     auto imm = src1->isA(Type::Int) ? safe_cast<int32_t>(src1->intVal()) :
1570                int32_t(src1->boolVal());
1571     v << subqi{imm, s0, d};
1572   } else {
1573     v << subq{s1, s0, d};
1574   }
1575 }
1576
1577 void CodeGenerator::cgMulInt(IRInstruction* inst) {
1578   // not using cgBinaryIntOp() here because x64 imul does not have
1579   // an immediate form. This means we can't provide a well-formed
1580   // Op class, and most of the complicated logic in cgBinaryIntOp
1581   // isn't necessary anyway.
1582   auto src0 = srcLoc(0).reg();
1583   auto src1 = srcLoc(1).reg();
1584   auto dst = dstLoc(0).reg();
1585   auto& v = vmain();
1586   assert(src0 != InvalidReg && src1 != InvalidReg && dst != InvalidReg);
1587   v << imul{src1, src0, dst};
1588 }
1589
1590 void CodeGenerator::cgAddDbl(IRInstruction* inst) {
1591   cgBinaryDblOp(inst, [&](Vout& v, VregXMM s0, VregXMM s1, VregXMM res) {
1592     v << addsd{s0, s1, res};
1593   });
1594 }
1595
1596 void CodeGenerator::cgSubDbl(IRInstruction* inst) {
1597   cgBinaryDblOp(inst, [&](Vout& v, VregXMM s0, VregXMM s1, VregXMM res) {
1598     v << subsd{s0, s1, res};
1599   });
1600 }
1601
1602 void CodeGenerator::cgMulDbl(IRInstruction* inst) {
1603   cgBinaryDblOp(inst, [&](Vout& v, VregXMM s0, VregXMM s1, VregXMM res) {
1604     v << mulsd{s0, s1, res};
1605   });
1606 }
1607
1608 void CodeGenerator::cgDivDbl(IRInstruction* inst) {
1609   const SSATmp* src1  = inst->src(0);
1610   const SSATmp* src2  = inst->src(1);
1611   auto loc1 = srcLoc(0);
1612   auto loc2 = srcLoc(1);
1613   auto exit = inst->taken();
1614
1615   auto dstReg  = dstLoc(0).reg();
1616   auto resReg  = dstReg.isSIMD() && dstReg != loc2.reg() ? dstReg :
1617                  PhysReg(rCgXMM0);
1618   assert(resReg.isSIMD());
1619
1620   // only load divisor
1621   PhysReg srcReg2 = prepXMMReg(m_as, src2, loc2, rCgXMM1);
1622   assert(srcReg2 != rCgXMM0);
1623
1624   // divide by zero check
1625   m_as.pxor(rCgXMM0, rCgXMM0);
1626   m_as.ucomisd(rCgXMM0, srcReg2);
1627   unlikelyIfBlock(CC_NP, [&] (Asm& a) {
1628     emitFwdJcc(a, CC_E, exit);
1629   });
1630
1631   // now load dividend
1632   PhysReg srcReg1 = prepXMMReg(m_as, src1, loc1, resReg);
1633   assert(srcReg1 != rCgXMM1);
1634
1635   emitMovRegReg(m_as, srcReg1, resReg);
1636   m_as.divsd(srcReg2, resReg);
1637   emitMovRegReg(m_as, resReg, dstReg);
1638 }
1639
1640 void CodeGenerator::cgAndInt(IRInstruction* inst) {
1641   cgBinaryIntOp<andq,andqi>(inst);
1642 }
1643
1644 void CodeGenerator::cgOrInt(IRInstruction* inst) {
1645   cgBinaryIntOp<orq,orqi>(inst);
1646 }
1647
1648 void CodeGenerator::cgXorInt(IRInstruction* inst) {
1649   if (inst->src(1)->isConst(-1)) {
1650     return cgUnaryIntOp<not>(dstLoc(0), inst->src(0), srcLoc(0));
1651   }
1652   cgBinaryIntOp<xorq,xorqi>(inst);
1653 }
1654
1655 void CodeGenerator::cgXorBool(IRInstruction* inst) {
1656   cgBinaryIntOp<xorb,xorbi>(inst);
1657 }
1658
1659 void CodeGenerator::cgMod(IRInstruction* inst) {
1660   static_assert(rCgGP != rax && rCgGP != rdx, "");
1661   auto const dstReg = dstLoc(0).reg();
1662   auto const reg0 = srcLoc(0).reg();
1663   auto const reg1 = srcLoc(1).reg();
1664   auto& v = vmain();
1665
1666   // spill rax and/or rdx
1667   bool spillRax = dstReg != rax;
1668   bool spillRdx = dstReg != rdx;
1669   if (spillRax) v << push{rax};
1670   if (spillRdx) v << push{rdx};
1671   // put divisor in rCgGP if it would get clobbered
1672   auto divisor = reg1 != rax && reg1 != rdx ? reg1 : PhysReg(rCgGP);
1673   v << copy{reg1, divisor};
1674   // put dividend in rax
1675   v << copy{reg0, rax};
1676   v << cqo{};            // sign-extend rax => rdx:rax
1677   v << idiv{divisor};    // rdx:rax/divisor => quot:rax, rem:rdx
1678   v << copy{rdx, dstReg};
1679   // restore rax and/or rdx
1680   if (spillRdx) v << pop{rdx};
1681   if (spillRax) v << pop{rax};
1682 }
1683
1684 void CodeGenerator::cgSqrt(IRInstruction* inst) {
1685   auto srcReg = srcLoc(0).reg();
1686   auto dstReg = dstLoc(0).reg();
1687   auto& v = vmain();
1688   auto tmp1 = srcReg.isSIMD() ? Vreg(srcReg) : v.makeReg();
1689   auto tmp2 = dstReg.isSIMD() ? Vreg(dstReg) : v.makeReg();
1690   v << copy{srcReg, tmp1};
1691   v << sqrtsd{tmp1, tmp2};
1692   v << copy{tmp2, dstReg};
1693 }
1694
1695 template<class Op, class Opi>
1696 void CodeGenerator::cgShiftCommon(IRInstruction* inst) {
1697   auto const srcReg0 = srcLoc(0).reg();
1698   auto const srcReg1 = srcLoc(1).reg();
1699   auto const dstReg  = dstLoc(0).reg();
1700   assert(srcReg0 != InvalidReg);
1701   auto& v = vmain();
1702
1703   // one immediate (right hand side)
1704   if (srcReg1 == InvalidReg) {
1705     v << copy{srcReg0, dstReg};
1706     v << Opi{safe_cast<int32_t>(inst->src(1)->intVal()), dstReg, dstReg};
1707     return;
1708   }
1709
1710   // in order to shift by a variable amount src2 must be in rcx :(
1711   bool swapRCX = srcReg1 != reg::rcx;
1712
1713   // will we be using dstReg as scratch storage?
1714   bool dstIsRHS = dstReg == srcReg1;
1715   bool tmpIsRCX = m_rScratch == reg::rcx;
1716   bool dstIsRCX = dstReg == reg::rcx;
1717
1718   // we need rcx for srcReg1 so we use srcReg1 as a temp for rcx, we also need
1719   // to handle the cases where the destination is rcx or src2 or both...
1720   auto resReg = dstIsRCX ? (dstIsRHS ? PhysReg(m_rScratch) : srcReg1)
1721                          : (dstIsRHS ? (tmpIsRCX ? dstReg : PhysReg(m_rScratch))
1722                                      : dstReg);
1723
1724   // if srcReg0 was in rcx it will be swapped with srcReg1 below
1725   auto regLeft = srcReg0 == reg::rcx ? srcReg1 : srcReg0;
1726
1727   // we use srcReg1 as a scratch for whatever is in rcx
1728   if (swapRCX) {
1729     v << copy2{reg::rcx, srcReg1, srcReg1, reg::rcx}; // emits xchgq
1730   }
1731
1732   v << copy{regLeft, resReg};
1733   v << Op{resReg, resReg};
1734
1735   if (resReg == dstReg && srcReg1 == dstReg) {
1736     // If we get here it means that m_rScratch was rcx and we shouldn't do any
1737     // more swapping because we stored the result in the right place
1738     return;
1739   }
1740
1741   if (swapRCX) {
1742     v << copy2{reg::rcx, srcReg1, srcReg1, reg::rcx};
1743   }
1744
1745   // if resReg == srcReg1 then dstReg must have been rcx and the above swap
1746   // already repaired the situation
1747   if (resReg != srcReg1) {
1748     v << copy{resReg, dstReg};
1749   }
1750 }
1751
1752 void CodeGenerator::cgShl(IRInstruction* inst) {
1753   cgShiftCommon<shlq,shlqi>(inst);
1754 }
1755
1756 void CodeGenerator::cgShr(IRInstruction* inst) {
1757   cgShiftCommon<sarq,sarqi>(inst);
1758 }
1759
1760 ///////////////////////////////////////////////////////////////////////////////
1761 // Comparison Operators
1762 ///////////////////////////////////////////////////////////////////////////////
1763
1764 #define DISPATCHER(name)\
1765   int64_t ccmp_ ## name (StringData* a1, StringData* a2)\
1766   { return name(a1, a2); }\
1767   int64_t ccmp_ ## name (StringData* a1, int64_t a2)\
1768   { return name(a1, a2); }\
1769   int64_t ccmp_ ## name (StringData* a1, ObjectData* a2)\
1770   { return name(a1, Object(a2)); }\
1771   int64_t ccmp_ ## name (ObjectData* a1, ObjectData* a2)\
1772   { return name(Object(a1), Object(a2)); }\
1773   int64_t ccmp_ ## name (ObjectData* a1, int64_t a2)\
1774   { return name(Object(a1), a2); }\
1775   int64_t ccmp_ ## name (ArrayData* a1, ArrayData* a2)\
1776   { return name(Array(a1), Array(a2)); }
1777
1778 DISPATCHER(same)
1779 DISPATCHER(equal)
1780 DISPATCHER(more)
1781 DISPATCHER(less)
1782
1783 #undef DISPATCHER
1784
1785 template <typename A, typename B>
1786 inline int64_t ccmp_nsame(A a, B b) { return !ccmp_same(a, b); }
1787
1788 template <typename A, typename B>
1789 inline int64_t ccmp_nequal(A a, B b) { return !ccmp_equal(a, b); }
1790
1791 // TODO Task #2661083: We cannot assume that "(a <= b) === !(a > b)" for
1792 // all types. In particular, this assumption does not hold when comparing
1793 // two arrays or comparing two objects. We should fix this.
1794 template <typename A, typename B>
1795 inline int64_t ccmp_lte(A a, B b) { return !ccmp_more(a, b); }
1796
1797 template <typename A, typename B>
1798 inline int64_t ccmp_gte(A a, B b) { return !ccmp_less(a, b); }
1799
1800 #define CG_OP_CMP(inst, cc, name)                                   \
1801   cgCmpHelper(inst, cc, ccmp_ ## name, ccmp_ ## name,               \
1802               ccmp_ ## name, ccmp_ ## name, ccmp_ ## name, ccmp_ ## name)
1803
1804 // SON - string, object, or number
1805 static bool typeIsSON(Type t) {
1806   return t.subtypeOfAny(Type::Str, Type::Obj, Type::Int, Type::Dbl);
1807 }
1808
1809 void CodeGenerator::cgCmpHelper(IRInstruction* inst, ConditionCode cc,
1810           int64_t (*str_cmp_str)(StringData*, StringData*),
1811           int64_t (*str_cmp_int)(StringData*, int64_t),
1812           int64_t (*str_cmp_obj)(StringData*, ObjectData*),
1813           int64_t (*obj_cmp_obj)(ObjectData*, ObjectData*),
1814           int64_t (*obj_cmp_int)(ObjectData*, int64_t),
1815           int64_t (*arr_cmp_arr)(ArrayData*,  ArrayData*)
1816         ) {
1817   SSATmp* src1  = inst->src(0);
1818   SSATmp* src2  = inst->src(1);
1819
1820   Type type1 = src1->type();
1821   Type type2 = src2->type();
1822
1823   auto loc1 = srcLoc(0);
1824   auto loc2 = srcLoc(1);
1825
1826   auto src1Reg = loc1.reg();
1827   auto src2Reg = loc2.reg();
1828   auto dstReg  = dstLoc(0).reg();
1829   auto& v = vmain();
1830
1831   // It is possible that some pass has been done after simplification; if such
1832   // a pass invalidates our invariants, then just punt.
1833
1834   // simplifyCmp has done const-const optimization
1835   //
1836   // If the types are the same and there is only one constant,
1837   // simplifyCmp has moved it to the right.
1838   if (src1->isConst()) {
1839     // TODO: #3626251 will let us eliminate this punt.
1840     CG_PUNT(cgOpCmpHelper_const);
1841   }
1842
1843   /////////////////////////////////////////////////////////////////////////////
1844   // case 1: null/string cmp string
1845   // simplifyCmp has converted the null to ""
1846   if (type1 <= Type::Str && type2 <= Type::Str) {
1847     cgCallHelper(v, CppCall::direct(str_cmp_str), callDest(inst),
1848       SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1849   }
1850
1851   /////////////////////////////////////////////////////////////////////////////
1852   // case 2: bool/null cmp anything
1853   // simplifyCmp has converted all args to bool
1854   else if (type1 <= Type::Bool && type2 <= Type::Bool) {
1855     if (src2->isConst()) {
1856       v << cmpbi{src2->boolVal(), src1Reg};
1857     } else {
1858       v << cmpb{src2Reg, src1Reg};
1859     }
1860     v << setcc{cc, dstReg};
1861   }
1862
1863   /////////////////////////////////////////////////////////////////////////////
1864   // case 3, 4, and 7: string/resource/object/number (sron) cmp sron
1865   // These cases must be amalgamated because Type::Obj can refer to an object
1866   //  or to a resource.
1867   // strings are canonicalized to the left, ints to the right
1868   else if (typeIsSON(type1) && typeIsSON(type2)) {
1869     if (type1 <= Type::Str) {
1870       // string cmp string is dealt with in case 1
1871       // string cmp double is punted above
1872
1873       if (type2 <= Type::Int) {
1874         cgCallHelper(v, CppCall::direct(str_cmp_int), callDest(inst),
1875                      SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1876       } else if (type2 <= Type::Obj) {
1877         cgCallHelper(v, CppCall::direct(str_cmp_obj), callDest(inst),
1878                      SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1879       } else {
1880         CG_PUNT(cgOpCmpHelper_sx);
1881       }
1882     }
1883
1884     else if (type1 <= Type::Obj) {
1885       // string cmp object is dealt with above
1886       // object cmp double is punted above
1887
1888       if (type2 <= Type::Obj) {
1889         cgCallHelper(v, CppCall::direct(obj_cmp_obj), callDest(inst),
1890                      SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1891       } else if (type2 <= Type::Int) {
1892         cgCallHelper(v, CppCall::direct(obj_cmp_int), callDest(inst),
1893                      SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1894       } else {
1895         CG_PUNT(cgOpCmpHelper_ox);
1896       }
1897     }
1898     else {
1899       CG_PUNT(cgOpCmpHelper_SON);
1900     }
1901   }
1902
1903   /////////////////////////////////////////////////////////////////////////////
1904   // case 5: array cmp array
1905   else if (type1 <= Type::Arr && type2 <= Type::Arr) {
1906     cgCallHelper(v, CppCall::direct(arr_cmp_arr),
1907       callDest(inst), SyncOptions::kSyncPoint, argGroup().ssa(0).ssa(1));
1908   }
1909
1910   /////////////////////////////////////////////////////////////////////////////
1911   // case 6: array cmp anything
1912   // simplifyCmp has already dealt with this case.
1913
1914   /////////////////////////////////////////////////////////////////////////////
1915   else {
1916     // We have a type which is not a common type. It might be a cell or a box.
1917     CG_PUNT(cgOpCmpHelper_unimplemented);
1918   }
1919 }
1920
1921 void CodeGenerator::cgEq(IRInstruction* inst) {
1922   CG_OP_CMP(inst, CC_E, equal);
1923 }
1924
1925 void CodeGenerator::cgEqX(IRInstruction* inst) {
1926   CG_OP_CMP(inst, CC_E, equal);
1927 }
1928
1929 void CodeGenerator::cgNeq(IRInstruction* inst) {
1930   CG_OP_CMP(inst, CC_NE, nequal);
1931 }
1932
1933 void CodeGenerator::cgNeqX(IRInstruction* inst) {
1934   CG_OP_CMP(inst, CC_NE, nequal);
1935 }
1936
1937 void CodeGenerator::cgSame(IRInstruction* inst) {
1938   CG_OP_CMP(inst, CC_E, same);
1939 }
1940
1941 void CodeGenerator::cgNSame(IRInstruction* inst) {
1942   CG_OP_CMP(inst, CC_NE, nsame);
1943 }
1944
1945 void CodeGenerator::cgLt(IRInstruction* inst) {
1946   CG_OP_CMP(inst, CC_L, less);
1947 }
1948
1949 void CodeGenerator::cgLtX(IRInstruction* inst) {
1950   CG_OP_CMP(inst, CC_L, less);
1951 }
1952
1953 void CodeGenerator::cgGt(IRInstruction* inst) {
1954   CG_OP_CMP(inst, CC_G, more);
1955 }
1956
1957 void CodeGenerator::cgGtX(IRInstruction* inst) {
1958   CG_OP_CMP(inst, CC_G, more);
1959 }
1960
1961 void CodeGenerator::cgLte(IRInstruction* inst) {
1962   CG_OP_CMP(inst, CC_LE, lte);
1963 }
1964
1965 void CodeGenerator::cgLteX(IRInstruction* inst) {
1966   CG_OP_CMP(inst, CC_LE, lte);
1967 }
1968
1969 void CodeGenerator::cgGte(IRInstruction* inst) {
1970   CG_OP_CMP(inst, CC_GE, gte);
1971 }
1972
1973 void CodeGenerator::cgGteX(IRInstruction* inst) {
1974   CG_OP_CMP(inst, CC_GE, gte);
1975 }
1976
1977 void CodeGenerator::emitCmpInt(IRInstruction* inst, ConditionCode cc) {
1978   auto dstReg = dstLoc(0).reg();
1979   emitCompareInt(inst);
1980   m_as.setcc(cc, rbyte(dstReg));
1981 }
1982
1983 void CodeGenerator::cgEqInt(IRInstruction* inst)  { emitCmpInt(inst, CC_E); }
1984 void CodeGenerator::cgNeqInt(IRInstruction* inst) { emitCmpInt(inst, CC_NE); }
1985 void CodeGenerator::cgLtInt(IRInstruction* inst)  { emitCmpInt(inst, CC_L); }
1986 void CodeGenerator::cgGtInt(IRInstruction* inst)  { emitCmpInt(inst, CC_G); }
1987 void CodeGenerator::cgLteInt(IRInstruction* inst) { emitCmpInt(inst, CC_LE); }
1988 void CodeGenerator::cgGteInt(IRInstruction* inst) { emitCmpInt(inst, CC_GE); }
1989
1990 void CodeGenerator::emitCmpEqDbl(IRInstruction* inst, ComparisonPred pred) {
1991   auto dstReg = dstLoc(0).reg();
1992   auto srcReg0 = prepXMMReg(m_as, inst->src(0), srcLoc(0), rCgXMM0);
1993   auto srcReg1 = prepXMMReg(m_as, inst->src(1), srcLoc(1), rCgXMM1);
1994
1995   m_as.  movsd   (srcReg1, rCgXMM2);
1996   m_as.  cmpsd   (srcReg0, rCgXMM2, pred);
1997   m_as.  movq_xr (rCgXMM2, dstReg);
1998   m_as.  andb    (1, rbyte(dstReg));
1999 }
2000
2001 void CodeGenerator::emitCmpRelDbl(IRInstruction* inst, ConditionCode cc,
2002                                   bool flipOperands) {
2003   auto dstReg = dstLoc(0).reg();
2004   auto srcReg0 = prepXMMReg(m_as, inst->src(0), srcLoc(0), rCgXMM0);
2005   auto srcReg1 = prepXMMReg(m_as, inst->src(1), srcLoc(1), rCgXMM1);
2006
2007   if (flipOperands) {
2008     std::swap(srcReg0, srcReg1);
2009   }
2010
2011   m_as.  ucomisd  (srcReg0, srcReg1);
2012   m_as.  setcc    (cc, rbyte(dstReg));
2013 }
2014
2015 void CodeGenerator::cgEqDbl(IRInstruction* inst)  {
2016   emitCmpEqDbl(inst, ComparisonPred::eq_ord);
2017 }
2018 void CodeGenerator::cgNeqDbl(IRInstruction* inst) {
2019   emitCmpEqDbl(inst, ComparisonPred::ne_unord);
2020 }
2021 void CodeGenerator::cgLtDbl(IRInstruction* inst)  {
2022   // This is a little tricky, because "unordered" is a thing.
2023   //
2024   //         ZF  PF  CF
2025   // x ?= y   1   1   1
2026   // x <  y   0   0   1
2027   // x == y   1   0   0
2028   // x >  y   0   0   0
2029   //
2030   // This trick lets us avoid needing to handle the unordered case specially.
2031   // The condition codes B and BE are true if CF == 1, which it is in the
2032   // unordered case, and that'll give incorrect results. So we just invert the
2033   // condition code (A and AE don't get set if CF == 1) and flip the operands.
2034   emitCmpRelDbl(inst, CC_A, true);
2035 }
2036 void CodeGenerator::cgGtDbl(IRInstruction* inst)  {
2037   emitCmpRelDbl(inst, CC_A, false);
2038 }
2039 void CodeGenerator::cgLteDbl(IRInstruction* inst) {
2040   emitCmpRelDbl(inst, CC_AE, true);
2041 }
2042 void CodeGenerator::cgGteDbl(IRInstruction* inst) {
2043   emitCmpRelDbl(inst, CC_AE, false);
2044 }
2045
2046 ///////////////////////////////////////////////////////////////////////////////
2047 // Type check operators
2048 ///////////////////////////////////////////////////////////////////////////////
2049
2050 // Overloads to put the {Object,Array}Data* into a register so
2051 // emitTypeTest can cmp to the Class*/ArrayKind expected by the
2052 // specialized Type
2053
2054 // Nothing to do, return the register that contain the ObjectData already
2055 Reg64 getDataPtrEnregistered(Asm& as, PhysReg dataSrc, Reg64 scratch) {
2056   return dataSrc;
2057 }
2058
2059 // Enregister the memoryRef so it can be used with an offset by the
2060 // cmp instruction
2061 Reg64 getDataPtrEnregistered(Asm& as, MemoryRef dataSrc, Reg64 scratch) {
2062   as.loadq(dataSrc, scratch);
2063   return scratch;
2064 }
2065
2066 template<class Loc1, class Loc2, class JmpFn>
2067 void CodeGenerator::emitTypeTest(Type type, Loc1 typeSrc, Loc2 dataSrc,
2068                                  JmpFn doJcc) {
2069   assert(!(type <= Type::Cls));
2070   ConditionCode cc;
2071   if (type <= Type::StaticStr) {
2072     emitCmpTVType(m_as, KindOfStaticString, typeSrc);
2073     cc = CC_E;
2074   } else if (type <= Type::Str) {
2075     assert(type != Type::CountedStr &&
2076            "We don't support guarding on CountedStr");
2077     emitTestTVType(m_as, KindOfStringBit, typeSrc);
2078     cc = CC_NZ;
2079   } else if (type == Type::Null) {
2080     emitCmpTVType(m_as, KindOfNull, typeSrc);
2081     cc = CC_LE;
2082   } else if (type == Type::UncountedInit) {
2083     emitTestTVType(m_as, KindOfUncountedInitBit, typeSrc);
2084     cc = CC_NZ;
2085   } else if (type == Type::Uncounted) {
2086     emitCmpTVType(m_as, KindOfRefCountThreshold, typeSrc);
2087     cc = CC_LE;
2088   } else if (type == Type::Cell) {
2089     assert(!m_curInst->is(LdRef));
2090     emitCmpTVType(m_as, KindOfRef, typeSrc);
2091     cc = CC_L;
2092   } else if (type == Type::Gen) {
2093     // nothing to check
2094     return;
2095   } else if (type == Type::InitCell) {
2096     assert(m_curInst->is(LdRef));
2097     // nothing to check: Refs cannot contain Uninit or another Ref.
2098     return;
2099   } else {
2100     always_assert(type.isKnownDataType());
2101     DataType dataType = type.toDataType();
2102     assert(dataType == KindOfRef ||
2103            (dataType >= KindOfUninit && dataType <= KindOfResource));
2104     emitCmpTVType(m_as, dataType, typeSrc);
2105     cc = CC_E;
2106   }
2107   doJcc(cc);
2108
2109   if (type.isSpecialized()) {
2110     emitSpecializedTypeTest(type, dataSrc, doJcc);
2111   }
2112 }
2113
2114 template<class DataLoc, class JmpFn>
2115 void CodeGenerator::emitSpecializedTypeTest(Type type, DataLoc dataSrc,
2116                                             JmpFn doJcc) {
2117   assert(type.isSpecialized());
2118   if (type < Type::Res) {
2119     // No cls field in Resource
2120     always_assert(0 && "unexpected guard on specialized Resource");
2121   }
2122
2123   if (type < Type::Obj) {
2124     // emit the specific class test
2125     assert(type.getClass()->attrs() & AttrNoOverride);
2126     auto reg = getDataPtrEnregistered(m_as, dataSrc, m_rScratch);
2127     emitCmpClass(m_as, type.getClass(), reg[ObjectData::getVMClassOffset()]);
2128     doJcc(CC_E);
2129   } else {
2130     assert(type < Type::Arr);
2131     auto reg = getDataPtrEnregistered(m_as, dataSrc, m_rScratch);
2132     m_as.cmpb(type.getArrayKind(), reg[ArrayData::offsetofKind()]);
2133     doJcc(CC_E);
2134   }
2135 }
2136
2137 template<class JmpFn>
2138 void CodeGenerator::emitIsTypeTest(IRInstruction* inst, JmpFn doJcc) {
2139   auto const src = inst->src(0);
2140   auto const loc = srcLoc(0);
2141
2142   // punt if specialized object for now
2143   if (inst->typeParam() < Type::Obj || inst->typeParam() < Type::Res) {
2144     CG_PUNT(IsType-SpecializedUnsupported);
2145   }
2146
2147   if (src->isA(Type::PtrToGen)) {
2148     PhysReg base = loc.reg();
2149     emitTypeTest(inst->typeParam(), base[TVOFF(m_type)],
2150                  base[TVOFF(m_data)], doJcc);
2151     return;
2152   }
2153   assert(src->isA(Type::Gen));
2154
2155   PhysReg typeSrcReg = loc.reg(1); // type register
2156   if (typeSrcReg == InvalidReg) {
2157     // Should only get here if the simplifier didn't run
2158     // TODO: #3626251 will handle this case.
2159     CG_PUNT(IsType-KnownType);
2160   }
2161   PhysReg dataSrcReg = loc.reg(0); // data register
2162   emitTypeTest(inst->typeParam(), typeSrcReg, dataSrcReg, doJcc);
2163 }
2164
2165 template<class Loc>
2166 void CodeGenerator::emitTypeCheck(Type type,
2167                                   Loc typeSrc,
2168                                   Loc dataSrc,
2169                                   Block* taken) {
2170   emitTypeTest(
2171     type, typeSrc, dataSrc,
2172     [&](ConditionCode cc) {
2173       emitFwdJcc(ccNegate(cc), taken);
2174     });
2175 }
2176
2177 template<class Loc>
2178 void CodeGenerator::emitTypeGuard(Type type, Loc typeSrc, Loc dataSrc) {
2179   emitTypeTest(type, typeSrc, dataSrc,
2180     [&](ConditionCode cc) {
2181       auto const destSK = SrcKey(curFunc(), m_unit.bcOff(), resumed());
2182       auto const destSR = mcg->tx().getSrcRec(destSK);
2183       destSR->emitFallbackJump(m_mainCode, ccNegate(cc));
2184     });
2185 }
2186
2187 void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) {
2188   m_as.setcc(cc, rbyte(dstLoc(0).reg()));
2189 }
2190
2191 void CodeGenerator::cgIsTypeMemCommon(IRInstruction* inst, bool negate) {
2192   bool called = false; // check emitSetCc is called only once
2193   emitIsTypeTest(inst,
2194     [&](ConditionCode cc) {
2195       assert(!called);
2196       emitSetCc(inst, negate ? ccNegate(cc) : cc);
2197       called = true;
2198     });
2199 }
2200
2201 void CodeGenerator::cgIsTypeCommon(IRInstruction* inst, bool negate) {
2202   bool called = false; // check emitSetCc is called only once
2203   emitIsTypeTest(inst,
2204     [&](ConditionCode cc) {
2205       assert(!called);
2206       emitSetCc(inst, negate ? ccNegate(cc) : cc);
2207       called = true;
2208     });
2209 }
2210
2211 void CodeGenerator::cgJmpIsTypeCommon(IRInstruction* inst, bool negate) {
2212   emitIsTypeTest(inst,
2213     [&](ConditionCode cc) {
2214       emitFwdJcc(negate ? ccNegate(cc) : cc, inst->taken());
2215     });
2216 }
2217
2218 void CodeGenerator::cgIsType(IRInstruction* inst) {
2219   cgIsTypeCommon(inst, false);
2220 }
2221
2222 void CodeGenerator::cgIsScalarType(IRInstruction* inst) {
2223   PhysReg typeReg = srcLoc(0).reg(1);
2224   PhysReg dstReg  = dstLoc(0).reg(0);
2225
2226   /* static asserts for KindOfBoolean <= scalar type <= KindOfString */
2227   static_assert(KindOfUninit < KindOfBoolean, "fix checks for IsScalar");
2228   static_assert(KindOfNull < KindOfBoolean, "fix checks for IsScalar");
2229   static_assert(KindOfInt64 > KindOfBoolean, "fix checks for IsScalar");
2230   static_assert(KindOfDouble > KindOfBoolean, "fix checks for IsScalar");
2231   static_assert(KindOfStaticString > KindOfBoolean, "fix checks for IsScalar");
2232   static_assert(KindOfString > KindOfBoolean, "fix checks for IsScalar");
2233
2234   static_assert(KindOfInt64 < KindOfString, "fix checks for IsScalar");
2235   static_assert(KindOfDouble < KindOfString, "fix checks for IsScalar");
2236   static_assert(KindOfStaticString < KindOfString, "fix checks for IsScalar");
2237   static_assert(KindOfArray > KindOfString, "fix checks for IsScalar");
2238   static_assert(KindOfObject > KindOfString, "fix checks for IsScalar");
2239   static_assert(KindOfResource > KindOfString, "fix checks for IsScalar");
2240
2241   static_assert(sizeof(DataType) == 1, "");
2242
2243   if (typeReg == InvalidReg) {
2244     auto const type = inst->src(0)->type();
2245     auto const imm = type <= (Type::Bool | Type::Int | Type::Dbl | Type::Str);
2246     m_as.    movl(Immed(imm), r32(dstReg));
2247     return;
2248   }
2249
2250   m_as.   movzbl(rbyte(typeReg), r32(dstReg));
2251   m_as.   subl(KindOfBoolean, r32(dstReg));
2252   m_as.   subl(KindOfString - KindOfBoolean + 1, r32(dstReg));
2253   m_as.   sbbl(r32(dstReg), r32(dstReg));
2254   m_as.   neg(dstReg);
2255 }
2256
2257 void CodeGenerator::cgIsNType(IRInstruction* inst) {
2258   cgIsTypeCommon(inst, true);
2259 }
2260
2261 void CodeGenerator::cgIsTypeMem(IRInstruction* inst) {
2262   cgIsTypeMemCommon(inst, false);
2263 }
2264
2265 void CodeGenerator::cgIsNTypeMem(IRInstruction* inst) {
2266   cgIsTypeMemCommon(inst, true);
2267 }
2268
2269 ///////////////////////////////////////////////////////////////////////////////
2270
2271 /*
2272  * Check instanceof using instance bitmasks.
2273  *
2274  * Note it's not necessary to check whether the test class is defined:
2275  * if it doesn't exist than the candidate can't be an instance of it
2276  * and will fail this check.
2277  */
2278 void CodeGenerator::emitInstanceBitmaskCheck(Vout& v, IRInstruction* inst) {
2279   auto const rObjClass     = srcLoc(0).reg(0);
2280   auto const testClassName = inst->src(1)->strVal();
2281   int offset;
2282   uint8_t mask;
2283   if (!InstanceBits::getMask(testClassName, offset, mask)) {
2284     always_assert(!"cgInstanceOfBitmask had no bitmask");
2285   }
2286   v << testbim{int8_t(mask), rObjClass[offset]};
2287 }
2288
2289 void CodeGenerator::emitInstanceBitmaskCheck(IRInstruction* inst) {
2290   emitInstanceBitmaskCheck(Vauto().main(m_as), inst);
2291 }
2292
2293 void CodeGenerator::cgInstanceOfBitmask(IRInstruction* inst) {
2294   auto& v = vmain();
2295   emitInstanceBitmaskCheck(v, inst);
2296   v << setcc{CC_NZ, dstLoc(0).reg()};
2297 }
2298
2299 void CodeGenerator::cgNInstanceOfBitmask(IRInstruction* inst) {
2300   auto& v = vmain();
2301   emitInstanceBitmaskCheck(v, inst);
2302   v << setcc{CC_Z, dstLoc(0).reg()};
2303 }
2304
2305 void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction* inst) {
2306   emitInstanceBitmaskCheck(inst);
2307   emitFwdJcc(CC_NZ, inst->taken());
2308 }
2309
2310 void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction* inst) {
2311   emitInstanceBitmaskCheck(inst);
2312   emitFwdJcc(CC_Z, inst->taken());
2313 }
2314
2315 void CodeGenerator::cgReqBindJmpInstanceOfBitmask(IRInstruction* inst) {
2316   auto& v = vmain();
2317   emitInstanceBitmaskCheck(v, inst);
2318   emitReqBindJcc(v, opToConditionCode(inst->op()),
2319                  inst->extra<ReqBindJccData>());
2320 }
2321
2322 void CodeGenerator::cgReqBindJmpNInstanceOfBitmask(IRInstruction* inst) {
2323   auto& v = vmain();
2324   emitInstanceBitmaskCheck(v, inst);
2325   emitReqBindJcc(v, opToConditionCode(inst->op()),
2326                  inst->extra<ReqBindJccData>());
2327 }
2328
2329 void CodeGenerator::cgSideExitJmpInstanceOfBitmask(IRInstruction* inst) {
2330   auto const extra = inst->extra<SideExitJccData>();
2331   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
2332   auto& v = vmain();
2333   emitInstanceBitmaskCheck(v, inst);
2334   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
2335 }
2336
2337 void CodeGenerator::cgSideExitJmpNInstanceOfBitmask(IRInstruction* inst) {
2338   auto const extra = inst->extra<SideExitJccData>();
2339   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
2340   auto& v = vmain();
2341   emitInstanceBitmaskCheck(v, inst);
2342   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
2343 }
2344
2345 void CodeGenerator::cgInstanceOf(IRInstruction* inst) {
2346   auto testReg = srcLoc(1).reg();
2347   auto destReg = dstLoc(0).reg();
2348   auto& v = vmain();
2349
2350   if (testReg == InvalidReg) {
2351     // Don't need to do the null check when the class is const.
2352     assert(inst->src(1)->clsVal() != nullptr);
2353     cgCallNative(v, inst);
2354     return;
2355   }
2356
2357   v << testq{testReg, testReg};
2358   ifThenElse(v, CC_NZ,
2359     [&](Vout& v) {
2360       cgCallNative(v, inst);
2361     },
2362     [&](Vout& v) {
2363       // testReg == 0, set dest to false (0)
2364       v << copy{testReg, destReg};
2365     }
2366   );
2367 }
2368
2369 /*
2370  * Check instanceof using the superclass vector on the end of the
2371  * Class entry.
2372  */
2373 void CodeGenerator::cgExtendsClass(IRInstruction* inst) {
2374   auto const rObjClass     = srcLoc(0).reg();
2375   auto const testClass     = inst->src(1)->clsVal();
2376   auto rTestClass          = srcLoc(1).reg();
2377   auto const rdst          = rbyte(dstLoc(0).reg());
2378   auto& a = m_as;
2379
2380   Label out;
2381   Label notExact;
2382   Label falseLabel;
2383
2384   if (rTestClass == InvalidReg) { // TODO(#2031606)
2385     rTestClass = m_rScratch; // careful below about asm-x64 smashing this
2386     emitLoadImm(a, (int64_t)testClass, rTestClass);
2387   }
2388
2389   // Test if it is the exact same class.  TODO(#2044801): we should be
2390   // doing this control flow at the IR level.
2391   if (!(testClass->attrs() & AttrAbstract)) {
2392     emitCmpClass(a, rTestClass, rObjClass);
2393     // If the test class cannot be extended, we only need to do the
2394     // exact check.
2395     if (testClass->attrs() & AttrNoOverride) {
2396       a.  sete(rdst);
2397       return;
2398     }
2399     a.    jne8   (notExact);
2400     a.    movb   (1, rdst);
2401     a.    jmp8   (out);
2402   }
2403
2404   auto const vecOffset = Class::classVecOff() +
2405     sizeof(LowClassPtr) * (testClass->classVecLen() - 1);
2406
2407   // Check the length of the class vectors---if the candidate's is at
2408   // least as long as the potential base (testClass) it might be a
2409   // subclass.
2410 asm_label(a, notExact);
2411   a.    cmpl   (safe_cast<int32_t>(testClass->classVecLen()),
2412                 rObjClass[Class::classVecLenOff()]);
2413   a.    jb8    (falseLabel);
2414
2415   // If it's a subclass, rTestClass must be at the appropriate index.
2416   emitCmpClass(a, rTestClass, rObjClass[vecOffset]);
2417   a.    sete   (rdst);
2418   a.    jmp8   (out);
2419
2420 asm_label(a, falseLabel);
2421   a.    xorl   (r32(rdst), r32(rdst));
2422
2423 asm_label(a, out);
2424 }
2425
2426 void CodeGenerator::cgConvDblToInt(IRInstruction* inst) {
2427   Vout& vmain = this->vmain();
2428   Vout& vcold = this->vcold();
2429
2430   auto src = inst->src(0);
2431   auto srcReg = prepXMM(vmain, src, srcLoc(0));
2432   auto dstReg = dstLoc(0).reg();
2433
2434   constexpr uint64_t maxULongAsDouble  = 0x43F0000000000000LL;
2435   constexpr uint64_t maxLongAsDouble   = 0x43E0000000000000LL;
2436
2437   auto rIndef = vmain.makeReg();
2438   vmain << ldimm{0x8000000000000000LL, rIndef};
2439   vmain << cvttsd2siq{srcReg, dstReg};
2440   vmain << cmpq{rIndef, dstReg};
2441
2442   unlikelyIfBlock(vmain, vcold, CC_E, [&] (Vout& v) {
2443     // result > max signed int or unordered
2444     auto tmp = v.makeReg();
2445     v << ldimm{0, tmp};
2446     v << ucomisd{tmp, srcReg};
2447     ifThen(v, CC_B, [&](Vout& v) {
2448       // src0 > 0 (CF = 1 -> less than 0 or unordered)
2449       ifThen(v, CC_NP, [&](Vout& v) {
2450         auto max_ulong = v.makeReg();
2451         v << ldimm{maxULongAsDouble, max_ulong};
2452         v << ucomisd{max_ulong, srcReg};
2453         ifThenElse(v, CC_B, [&](Vout& v) {
2454           // src0 > ULONG_MAX
2455           v << ldimm{0, dstReg};
2456         }, [&](Vout& v) {
2457           // 0 < src0 <= ULONG_MAX
2458           auto max_long = v.makeReg();
2459           v << ldimm{maxLongAsDouble, max_long};
2460
2461           // we know that LONG_MAX < src0 <= UINT_MAX, therefore,
2462           // 0 < src0 - ULONG_MAX <= LONG_MAX
2463           auto tmp_sub = v.makeReg();
2464           v << subsd{max_long, srcReg, tmp_sub};
2465           v << cvttsd2siq{tmp_sub, dstReg};
2466
2467           // We want to simulate integer overflow so we take the resulting
2468           // integer and flip its sign bit (NB: we don't use orq here
2469           // because it's possible that src0 == LONG_MAX in which case
2470           // cvttsd2siq will yeild an indefiniteInteger, which we would
2471           // like to make zero)
2472           v << xorq{rIndef, dstReg, dstReg};
2473         });
2474       });
2475     });
2476   });
2477 }
2478
2479 void CodeGenerator::cgConvDblToBool(IRInstruction* inst) {
2480   auto dstReg = dstLoc(0).reg();
2481   auto srcReg = srcLoc(0).reg();
2482   emitMovRegReg(m_as, srcReg, dstReg);
2483   m_as.shlq(1, dstReg); // 0.0 stays zero and -0.0 is now 0.0
2484   m_as.setne(rbyte(dstReg)); // lower byte becomes 1 if dstReg != 0
2485   m_as.movzbl(rbyte(dstReg), r32(dstReg));
2486 }
2487
2488 void CodeGenerator::cgConvIntToBool(IRInstruction* inst) {
2489   auto dstReg = dstLoc(0).reg();
2490   auto srcReg = srcLoc(0).reg();
2491   m_as.testq(srcReg, srcReg);
2492   m_as.setne(rbyte(dstReg));
2493   m_as.movzbl(rbyte(dstReg), r32(dstReg));
2494 }
2495
2496 void CodeGenerator::cgConvArrToBool(IRInstruction* inst) {
2497   auto dstReg = dstLoc(0).reg();
2498   auto srcReg = srcLoc(0).reg();
2499
2500   // This will incorrectly result in "true" for a NameValueTableWrapper that is
2501   // empty. You can only get such a thing through very contrived PHP, so the
2502   // savings of a branch and a block of cold code outweights the edge-case bug.
2503   m_as.    cmpl  (0, srcReg[ArrayData::offsetofSize()]);
2504   m_as.    setcc (CC_NZ, rbyte(dstReg));
2505 }
2506
2507 /*
2508  * emit something equivalent to testl(val, mr),
2509  * but with a shorter encoding (eg testb(val, mr))
2510  * if possible.
2511  */
2512 void testimm(Asm& as, uint32_t val, MemoryRef mr) {
2513   int off = 0;
2514   auto v = val;
2515   while (v > 0xff && !(v & 0xff)) {
2516     off++;
2517     v >>= 8;
2518   }
2519   if (v > 0xff) {
2520     as.testl((int32_t)val, mr);
2521   } else {
2522     as.testb((int8_t)v, *(mr.r + off));
2523   }
2524 }
2525
2526 void CodeGenerator::cgColIsEmpty(IRInstruction* inst) {
2527   DEBUG_ONLY auto const ty = inst->src(0)->type();
2528   assert(ty < Type::Obj &&
2529          ty.getClass() &&
2530          ty.getClass()->isCollectionClass());
2531   auto& a = m_as;
2532   a.    cmpl   (0, srcLoc(0).reg()[FAST_COLLECTION_SIZE_OFFSET]);
2533   a.    sete   (rbyte(dstLoc(0).reg()));
2534 }
2535
2536 void CodeGenerator::cgColIsNEmpty(IRInstruction* inst) {
2537   DEBUG_ONLY auto const ty = inst->src(0)->type();
2538   assert(ty < Type::Obj &&
2539          ty.getClass() &&
2540          ty.getClass()->isCollectionClass());
2541   auto& a = m_as;
2542   a.    cmpl   (0, srcLoc(0).reg()[FAST_COLLECTION_SIZE_OFFSET]);
2543   a.    setne  (rbyte(dstLoc(0).reg()));
2544 }
2545
2546 void CodeGenerator::cgConvObjToBool(IRInstruction* inst) {
2547   auto const rdst = dstLoc(0).reg();
2548   auto const rsrc = srcLoc(0).reg();
2549   auto& a = m_as;
2550
2551   testimm(a, ObjectData::CallToImpl, rsrc[ObjectData::attributeOff()]);
2552   unlikelyIfThenElse(
2553     CC_NZ,
2554     [&] (Asm& a) {
2555       testimm(a,
2556               ObjectData::IsCollection,
2557               rsrc[ObjectData::attributeOff()]);
2558       ifThenElse(
2559         a,
2560         CC_NZ,
2561         [&] (Asm& a) { // rsrc points to native collection
2562           a.cmpl(0, rsrc[FAST_COLLECTION_SIZE_OFFSET]);
2563           a.setne(rbyte(rdst)); // true iff size not zero
2564         },
2565         [&] (Asm& a) { // rsrc is not a native collection
2566           cgCallHelper(
2567             a,
2568             CppCall::method(&ObjectData::o_toBoolean),
2569             callDest(inst),
2570             SyncOptions::kSyncPoint,
2571             argGroup().ssa(0));
2572         }
2573       );
2574     },
2575     [&] (Asm& a) {
2576       a.    movb  (1, rbyte(rdst));
2577     }
2578   );
2579 }
2580
2581 void CodeGenerator::emitConvBoolOrIntToDbl(IRInstruction* inst) {
2582   SSATmp* src = inst->src(0);
2583   assert(src->isA(Type::Bool) || src->isA(Type::Int));
2584   auto dstReg = dstLoc(0).reg();
2585   auto srcReg = srcLoc(0).reg();
2586   // cvtsi2sd doesn't modify the high bits of its target, which can
2587   // cause false dependencies to prevent register renaming from kicking
2588   // in. Break the dependency chain by zeroing out the XMM reg.
2589   auto& v = vmain();
2590   zeroExtendIfBool(v, src, srcReg);
2591   auto tmp_dbl = v.makeReg();
2592   v << cvtsi2sd{srcReg, tmp_dbl};
2593   v << copy{tmp_dbl, dstReg};
2594 }
2595
2596 void CodeGenerator::cgConvBoolToDbl(IRInstruction* inst) {
2597   emitConvBoolOrIntToDbl(inst);
2598 }
2599
2600 void CodeGenerator::cgConvIntToDbl(IRInstruction* inst) {
2601   emitConvBoolOrIntToDbl(inst);
2602 }
2603
2604 void CodeGenerator::cgConvBoolToInt(IRInstruction* inst) {
2605   auto dstReg = dstLoc(0).reg();
2606   auto srcReg = srcLoc(0).reg();
2607   m_as.movzbl(rbyte(srcReg), r32(dstReg));
2608 }
2609
2610 void CodeGenerator::cgConvBoolToStr(IRInstruction* inst) {
2611   auto dstReg = dstLoc(0).reg();
2612   auto srcReg = srcLoc(0).reg();
2613   m_as.testb(rbyte(srcReg), rbyte(srcReg));
2614   m_as.movq(makeStaticString(""), dstReg);
2615   m_as.movq(makeStaticString("1"), m_rScratch);
2616   m_as.cmov_reg64_reg64(CC_NZ, m_rScratch, dstReg);
2617 }
2618
2619 void CodeGenerator::cgConvClsToCctx(IRInstruction* inst) {
2620   auto const sreg = srcLoc(0).reg();
2621   auto const dreg = dstLoc(0).reg();
2622   auto& a = m_as;
2623   emitMovRegReg(a, sreg, dreg);
2624   a.    orq   (1, dreg);
2625 }
2626
2627 void CodeGenerator::cgUnboxPtr(IRInstruction* inst) {
2628   auto srcReg = srcLoc(0).reg();
2629   auto dstReg = dstLoc(0).reg();
2630   assert(srcReg != InvalidReg);
2631   emitMovRegReg(m_as, srcReg, dstReg);
2632   emitDerefIfVariant(m_as, PhysReg(dstReg));
2633 }
2634
2635 void CodeGenerator::cgLdFuncCachedCommon(IRInstruction* inst) {
2636   auto const dst  = dstLoc(0).reg();
2637   auto const name = inst->extra<LdFuncCachedData>()->name;
2638   auto const ch   = NamedEntity::get(name)->getFuncHandle();
2639   auto& a = m_as;
2640
2641   if (dst == InvalidReg) {
2642     a.   cmpq  (0, rVmTl[ch]);
2643   } else {
2644     a.   loadq (rVmTl[ch], dst);
2645     a.   testq (dst, dst);
2646   }
2647 }
2648
2649 void CodeGenerator::cgLdFuncCached(IRInstruction* inst) {
2650   cgLdFuncCachedCommon(inst);
2651   unlikelyIfBlock(CC_Z, [&] (Asm& a) {
2652     const Func* (*const func)(const StringData*) = lookupUnknownFunc;
2653     cgCallHelper(
2654       a,
2655       CppCall::direct(func),
2656       callDest(inst),
2657       SyncOptions::kSyncPoint,
2658       argGroup()
2659         .immPtr(inst->extra<LdFuncCached>()->name)
2660     );
2661   });
2662 }
2663
2664 void CodeGenerator::cgLdFuncCachedSafe(IRInstruction* inst) {
2665   cgLdFuncCachedCommon(inst);
2666   if (auto const taken = inst->taken()) {
2667     emitFwdJcc(m_as, CC_Z, taken);
2668   }
2669 }
2670
2671 void CodeGenerator::cgLdFuncCachedU(IRInstruction* inst) {
2672   auto const dstReg    = dstLoc(0).reg();
2673   auto const extra     = inst->extra<LdFuncCachedU>();
2674   auto const hFunc     = NamedEntity::get(extra->name)->getFuncHandle();
2675
2676   auto& a = m_as;
2677
2678   // Check the first function handle, otherwise try to autoload.
2679   if (dstReg == InvalidReg) {
2680     a.   cmpq  (0, rVmTl[hFunc]);
2681   } else {
2682     a.   loadq (rVmTl[hFunc], dstReg);
2683     a.   testq (dstReg, dstReg);
2684   }
2685
2686   unlikelyIfBlock(CC_Z, [&] (Asm& a) {
2687     // If we get here, things are going to be slow anyway, so do all the
2688     // autoloading logic in lookupFallbackFunc instead of ASM
2689     const Func* (*const func)(const StringData*, const StringData*) =
2690         lookupFallbackFunc;
2691     cgCallHelper(
2692       a,
2693       CppCall::direct(func),
2694       callDest(inst),
2695       SyncOptions::kSyncPoint,
2696       argGroup()
2697         .immPtr(extra->name)
2698         .immPtr(extra->fallback)
2699     );
2700   });
2701 }
2702
2703 void CodeGenerator::cgLdFunc(IRInstruction* inst) {
2704   auto const ch = FuncCache::alloc();
2705   RDS::recordRds(ch, sizeof(FuncCache),
2706                  "FuncCache", curFunc()->fullName()->data());
2707
2708   // raises an error if function not found
2709   cgCallHelper(m_as,
2710                CppCall::direct(FuncCache::lookup),
2711                callDest(dstLoc(0).reg()),
2712                SyncOptions::kSyncPoint,
2713                argGroup().imm(ch).ssa(0/*methodName*/));
2714 }
2715
2716 void CodeGenerator::cgLdObjClass(IRInstruction* inst) {
2717   auto dstReg = dstLoc(0).reg();
2718   auto objReg = srcLoc(0).reg();
2719   emitLdObjClass(m_as, objReg, dstReg);
2720 }
2721
2722 void CodeGenerator::cgLdObjMethod(IRInstruction* inst) {
2723   assert(inst->taken() && inst->taken()->isCatch()); // must have catch block
2724   using namespace MethodCache;
2725
2726   auto const clsReg    = srcLoc(0).reg();
2727   auto const actRecReg = srcLoc(1).reg();
2728   auto const extra     = inst->extra<LdObjMethodData>();
2729   auto& v = vmain();
2730
2731   auto const handle = RDS::alloc<Entry, sizeof(Entry)>().handle();
2732   if (RuntimeOption::EvalPerfDataMap) {
2733     auto const caddr_hand = reinterpret_cast<char*>(
2734       static_cast<intptr_t>(handle)
2735     );
2736     Debug::DebugInfo::recordDataMap(
2737       caddr_hand,
2738       caddr_hand + sizeof(TypedValue),
2739       folly::format("rds+MethodCache-{}",
2740         curFunc()->fullName()->data()).str());
2741   }
2742
2743   auto const mcHandler = extra->fatal ? handlePrimeCacheInit<true>
2744                                       : handlePrimeCacheInit<false>;
2745
2746   auto fast_path = v.makeBlock();
2747   auto slow_path = v.makeBlock();
2748   auto done = v.makeBlock();
2749
2750   /*
2751    * Inline cache: we "prime" the cache across requests by smashing
2752    * this immediate to hold a Func* in the upper 32 bits, and a Class*
2753    * in the lower 32 bits.  (If both are low-malloced pointers can
2754    * fit.)  See pmethodCacheMissPath.
2755    */
2756   v << mcprep{rAsm};
2757   v << movq{rAsm, m_rScratch};
2758   v << movl{Vreg{rAsm}, Vreg{rAsm}};  // zeros the top 32 bits
2759   v << cmpq{rAsm, clsReg};
2760   v << jcc{CC_NE, {fast_path, slow_path}};
2761
2762   v = fast_path;
2763   v << shrqi{32, m_rScratch, m_rScratch};
2764   v << storeq{m_rScratch, actRecReg[AROFF(m_func)]};
2765   v << jmp{done};
2766
2767   v = slow_path;
2768   cgCallHelper(v,
2769     CppCall::direct(mcHandler),
2770     kVoidDest,
2771     SyncOptions::kSmashableAndSyncPoint,
2772     argGroup()
2773       .addr(rVmTl, safe_cast<int32_t>(handle))
2774       .ssa(1/*actRec*/)
2775       .immPtr(extra->method)
2776       .ssa(0/*cls*/)
2777       .immPtr(curClass())
2778       // The scratch reg contains the prime data before we've smashed the call
2779       // to handleSlowPath.  After, it contains the primed Class/Func pair.
2780       .reg(m_rScratch)
2781   );
2782   v << jmp{done};
2783   v = done;
2784 }
2785
2786 void CodeGenerator::cgLdObjInvoke(IRInstruction* inst) {
2787   auto const rsrc = srcLoc(0).reg();
2788   auto const rdst = dstLoc(0).reg();
2789   auto& a = m_as;
2790
2791   a.   loadq  (rsrc[Class::invokeOff()], rdst);
2792   a.   testq  (rdst, rdst);
2793   emitFwdJcc  (a, CC_Z, inst->taken());
2794 }
2795
2796 void CodeGenerator::cgStRetVal(IRInstruction* inst) {
2797   auto  const rFp = srcLoc(0).reg();
2798   auto* const val = inst->src(1);
2799   cgStore(rFp[AROFF(m_r)], val, srcLoc(1), Width::Full);
2800 }
2801
2802 void CodeGenerator::cgRetAdjustStack(IRInstruction* inst) {
2803   auto const rFp   = srcLoc(0).reg();
2804   auto const dstSp = dstLoc(0).reg();
2805   auto& a = m_as;
2806   a.    lea   (rFp[AROFF(m_r)], dstSp);
2807 }
2808
2809 void CodeGenerator::cgLdRetAddr(IRInstruction* inst) {
2810   auto fpReg = srcLoc(0).reg(0);
2811   assert(fpReg != InvalidReg);
2812   m_as.push(fpReg[AROFF(m_savedRip)]);
2813 }
2814
2815 void traceRet(ActRec* fp, Cell* sp, void* rip) {
2816   if (rip == mcg->tx().uniqueStubs.callToExit) {
2817     return;
2818   }
2819   checkFrame(fp, sp, /*fullCheck*/ false, 0);
2820   assert(sp <= (Cell*)fp || fp->resumed());
2821   // check return value if stack not empty
2822   if (sp < (Cell*)fp) assertTv(sp);
2823 }
2824
2825 void CodeGenerator::emitTraceRet(Asm& a) {
2826   // call to a trace function
2827   a.    movq  (rVmFp, rdi);
2828   a.    movq  (rVmSp, rsi);
2829   a.    loadq (*rsp, rdx); // return ip from native stack
2830   // do the call; may use a trampoline
2831   emitCall(a, TCA(traceRet));
2832 }
2833
2834 void CodeGenerator::cgRetCtrl(IRInstruction* inst) {
2835   // Make sure rVmFp and rVmSp are set appropriately
2836   emitMovRegReg(m_as, srcLoc(0/*sp*/).reg(), rVmSp);
2837   emitMovRegReg(m_as, srcLoc(1/*fp*/).reg(), rVmFp);
2838
2839   // Return control to caller
2840   if (RuntimeOption::EvalHHIRGenerateAsserts) {
2841     emitTraceRet(m_as);
2842   }
2843
2844   m_as.ret();
2845   if (RuntimeOption::EvalHHIRGenerateAsserts) {
2846     m_as.ud2();
2847   }
2848 }
2849
2850 void CodeGenerator::emitReqBindAddr(TCA& dest,
2851                                     SrcKey sk) {
2852   mcg->setJmpTransID((TCA)&dest);
2853
2854   dest = emitEphemeralServiceReq(m_frozenCode,
2855                                  mcg->getFreeStub(m_frozenCode,
2856                                                   &mcg->cgFixups()),
2857                                  REQ_BIND_ADDR,
2858                                  &dest,
2859                                  sk.toAtomicInt(),
2860                                  TransFlags{}.packed);
2861   mcg->cgFixups().m_codePointers.insert(&dest);
2862 }
2863
2864 void CodeGenerator::cgLdBindAddr(IRInstruction* inst) {
2865   auto data   = inst->extra<LdBindAddr>();
2866   auto dstReg = dstLoc(0).reg();
2867
2868   // Emit service request to smash address of SrcKey into 'addr'.
2869   TCA* addrPtr = mcg->allocData<TCA>(sizeof(TCA), 1);
2870   emitReqBindAddr(*addrPtr, data->sk);
2871
2872   // Load the maybe bound address.
2873   auto addr = intptr_t(addrPtr);
2874   // the tc/global data is intentionally layed out to guarantee
2875   // rip-relative addressing will work.
2876   // Also, a rip-relative load, is 1 byte smaller than the corresponding
2877   // baseless load.
2878   m_as.loadq(rip[addr], dstReg);
2879 }
2880
2881 void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) {
2882   JmpSwitchData* data = inst->extra<JmpSwitchDest>();
2883   SSATmp* index       = inst->src(0);
2884   auto indexReg       = srcLoc(0).reg();
2885   auto rTmp           = m_rScratch;
2886
2887   if (!index->isConst()) {
2888     if (data->bounded) {
2889       if (data->base) {
2890         if (deltaFits(data->base, sz::dword)) {
2891           m_as.  subq(safe_cast<int32_t>(data->base), indexReg);
2892         } else {
2893           m_as.  emitImmReg(data->base, rTmp);
2894           m_as.  subq(rTmp, indexReg);
2895         }
2896       }
2897       m_as.    cmpq(data->cases - 2, indexReg);
2898       mcg->backEnd().prepareForSmash(m_mainCode, kJmpccLen);
2899       TCA def = emitEphemeralServiceReq(
2900         m_frozenCode,
2901         mcg->getFreeStub(m_frozenCode, &mcg->cgFixups()),
2902         REQ_BIND_JMPCC_SECOND,
2903         RipRelative(m_as.frontier()),
2904         data->defaultOff,
2905         CC_AE);
2906       mcg->setJmpTransID(m_as.frontier());
2907
2908       m_as.    jae(def);
2909     }
2910
2911     TCA* table = mcg->allocData<TCA>(sizeof(TCA), data->cases);
2912     m_as.   lea(rip[(intptr_t)table], m_rScratch);
2913     assert(((int32_t*)m_as.frontier())[-1] + m_as.frontier() == (TCA)table);
2914     m_as.   jmp(m_rScratch[indexReg*8]);
2915
2916     for (int i = 0; i < data->cases; i++) {
2917       emitReqBindAddr(table[i], SrcKey(curFunc(), data->targets[i], resumed()));
2918     }
2919   } else {
2920     int64_t indexVal = index->intVal();
2921
2922     if (data->bounded) {
2923       indexVal -= data->base;
2924       if (indexVal >= data->cases - 2 || indexVal < 0) {
2925         emitBindJmp(m_mainCode, m_frozenCode,
2926                     SrcKey(curFunc(), data->defaultOff, resumed()));
2927         return;
2928       }
2929     }
2930     emitBindJmp(m_mainCode, m_frozenCode,
2931                 SrcKey(curFunc(), data->targets[indexVal], resumed()));
2932   }
2933 }
2934
2935 void CodeGenerator::cgLdSSwitchDestFast(IRInstruction* inst) {
2936   auto data = inst->extra<LdSSwitchDestFast>();
2937
2938   auto table = mcg->allocData<SSwitchMap>(64);
2939   new (table) SSwitchMap(data->numCases);
2940   for (int64_t i = 0; i < data->numCases; ++i) {
2941     table->add(data->cases[i].str, nullptr);
2942     TCA* addr = table->find(data->cases[i].str);
2943     emitReqBindAddr(*addr, SrcKey(curFunc(), data->cases[i].dest, resumed()));
2944   }
2945   TCA* def = mcg->allocData<TCA>(sizeof(TCA), 1);
2946   emitReqBindAddr(*def, SrcKey(curFunc(), data->defaultOff, resumed()));
2947
2948   cgCallHelper(m_as,
2949                CppCall::direct(sswitchHelperFast),
2950                callDest(inst),
2951                SyncOptions::kNoSyncPoint,
2952                argGroup()
2953                  .ssa(0)
2954                  .immPtr(table)
2955                  .immPtr(def));
2956 }
2957
2958 static TCA sswitchHelperSlow(TypedValue typedVal,
2959                              const StringData** strs,
2960                              int numStrs,
2961                              TCA* jmptab) {
2962   Cell* cell = tvToCell(&typedVal);
2963   for (int i = 0; i < numStrs; ++i) {
2964     if (cellEqual(*cell, strs[i])) return jmptab[i];
2965   }
2966   return jmptab[numStrs]; // default case
2967 }
2968
2969 void CodeGenerator::cgLdSSwitchDestSlow(IRInstruction* inst) {
2970   auto data = inst->extra<LdSSwitchDestSlow>();
2971
2972   auto strtab = mcg->allocData<const StringData*>(
2973     sizeof(const StringData*), data->numCases);
2974   auto jmptab = mcg->allocData<TCA>(sizeof(TCA), data->numCases + 1);
2975   for (int i = 0; i < data->numCases; ++i) {
2976     strtab[i] = data->cases[i].str;
2977     emitReqBindAddr(jmptab[i],
2978                     SrcKey(curFunc(), data->cases[i].dest, resumed()));
2979   }
2980   emitReqBindAddr(jmptab[data->numCases],
2981                   SrcKey(curFunc(), data->defaultOff, resumed()));
2982
2983   cgCallHelper(m_as,
2984                CppCall::direct(sswitchHelperSlow),
2985                callDest(inst),
2986                SyncOptions::kSyncPoint,
2987                argGroup()
2988                  .typedValue(0)
2989                  .immPtr(strtab)
2990                  .imm(data->numCases)
2991                  .immPtr(jmptab));
2992 }
2993
2994 /*
2995  * It'd be nice not to have the cgMov here (and just copy propagate
2996  * the source or something), but for now we're keeping it allocated to
2997  * rVmFp so inlined calls to C++ helpers that use the rbp chain to
2998  * find the caller's ActRec will work correctly.
2999  *
3000  * This instruction primarily exists to assist in optimizing away
3001  * unused activation records, so it's usually not going to happen
3002  * anyway.
3003  */
3004 void CodeGenerator::cgDefInlineFP(IRInstruction* inst) {
3005   auto const calleeFP = srcLoc(0).reg();
3006   auto const callerFP = srcLoc(2).reg();
3007   auto const fakeRet  = mcg->tx().uniqueStubs.retInlHelper;
3008   auto const retBCOff = inst->extra<DefInlineFP>()->retBCOff;
3009
3010   m_as.    storeq (callerFP, calleeFP[AROFF(m_sfp)]);
3011   emitImmStoreq(m_as, intptr_t(fakeRet), calleeFP[AROFF(m_savedRip)]);
3012   m_as.    storel (retBCOff, calleeFP[AROFF(m_soff)]);
3013   cgMov(inst);
3014 }
3015
3016 void CodeGenerator::cgInlineReturn(IRInstruction* inst) {
3017   auto fpReg = srcLoc(0).reg();
3018   assert(fpReg == rVmFp);
3019   m_as.    loadq  (fpReg[AROFF(m_sfp)], rVmFp);
3020 }
3021
3022 void CodeGenerator::cgReDefSP(IRInstruction* inst) {
3023   // TODO(#2288359): this instruction won't be necessary (for
3024   // non-generator frames) when we don't track rVmSp independently
3025   // from rVmFp.  In generator frames we'll have to track offsets from
3026   // a DefResumableSP or something similar.
3027   auto fp  = srcLoc(1).reg();
3028   auto dst = dstLoc(0).reg();
3029   auto off = -inst->extra<ReDefSP>()->spOffset * sizeof(Cell);
3030   emitLea(m_as, fp[off], dst);
3031 }
3032
3033 void CodeGenerator::cgFreeActRec(IRInstruction* inst) {
3034   auto ptr = srcLoc(0).reg();
3035   auto off = AROFF(m_sfp);
3036   auto dst = dstLoc(0).reg();
3037   m_as.loadq(ptr[off], dst);
3038 }
3039
3040 void emitSpill(Asm& as, const PhysLoc& s, const PhysLoc& d, Type t) {
3041   assert(s.numWords() == d.numWords());
3042   assert(!s.spilled() && d.spilled());
3043   if (s.isFullSIMD()) {
3044     as.movdqu(s.reg(0), reg::rsp[d.offset(0)]);
3045   } else {
3046     for (int i = 0, n = s.numAllocated(); i < n; ++i) {
3047       // store the whole register even if it holds a bool or DataType
3048       emitStoreReg(as, s.reg(i), reg::rsp[d.offset(i)]);
3049     }
3050   }
3051 }
3052
3053 void emitReload(Asm& as, const PhysLoc& s, const PhysLoc& d, Type t) {
3054   assert(s.numWords() == d.numWords());
3055   assert(s.spilled() && !d.spilled());
3056   if (d.isFullSIMD()) {
3057     as.movdqu(reg::rsp[s.offset(0)], d.reg(0));
3058   } else {
3059     for (int i = 0, n = d.numAllocated(); i < n; ++i) {
3060       // load the whole register even if it holds a bool or DataType
3061       emitLoadReg(as, reg::rsp[s.offset(i)], d.reg(i));
3062     }
3063   }
3064 }
3065
3066 void CodeGenerator::cgShuffle(IRInstruction* inst) {
3067   // Each destination is unique, there are no mem-mem copies, and
3068   // there are no cycles involving spill slots.  So do the shuffling
3069   // in this order:
3070   // 1. reg->mem (stores)
3071   // 2. reg->reg (parallel copies)
3072   // 3. mem->reg (loads) & imm->reg (constants)
3073   PhysReg::Map<PhysReg> moves;    // moves[dst] = src
3074   for (uint32_t i = 0, n = inst->numSrcs(); i < n; ++i) {
3075     auto& rd = inst->extra<Shuffle>()->dests[i];
3076     if (rd.numAllocated() == 0) continue; // ignore unused dests.
3077     auto src = inst->src(i);
3078     auto rs = srcLoc(i);
3079     if (rd.spilled()) {
3080       emitSpill(m_as, rs, rd, src->type());
3081     } else if (!rs.spilled()) {
3082       auto s0 = rs.reg(0);
3083       auto d0 = rd.reg(0);
3084       if (s0 != InvalidReg) moves[d0] = s0;
3085       auto s1 = rs.reg(1);
3086       auto d1 = rd.reg(1);
3087       if (s1 != InvalidReg) moves[d1] = s1;
3088     }
3089   }
3090   // Compute a serial order of moves and swaps.  We can use m_rScratch
3091   // here since cgShuffle is a standalone HHIR instruction, and sometimes
3092   // its a low-numbered register.
3093   auto rTmp = m_rScratch;
3094   auto howTo = doRegMoves(moves, rTmp);
3095   for (auto& how : howTo) {
3096     if (how.m_kind == MoveInfo::Kind::Move) {
3097       emitMovRegReg(m_as, how.m_src, how.m_dst);
3098     } else {
3099       // do swap - only support GPRs
3100       assert(how.m_src.isGP() && how.m_dst.isGP());
3101       m_as.xchgq(how.m_src, how.m_dst);
3102     }
3103   }
3104   // now do reg<-mem loads and reg<-imm moves. We have already
3105   // dealt with stores, moves, and swaps, so all rTmp is available
3106   // even if it originally contained a value.
3107   for (uint32_t i = 0, n = inst->numSrcs(); i < n; ++i) {
3108     auto src = inst->src(i);
3109     auto rs = srcLoc(i);
3110     auto& rd = inst->extra<Shuffle>()->dests[i];
3111     if (rd.numAllocated() == 0) continue; // ignore unused dests.
3112     if (rd.spilled()) continue;
3113     if (rs.spilled()) {
3114       emitReload(m_as, rs, rd, src->type());
3115       continue;
3116     }
3117     if (rs.numAllocated() == 0) {
3118       assert(src->isConst());
3119       auto r = rd.reg(0);
3120       auto imm = src->type().needsValueReg() ? src->rawVal() :
3121                  0xdeadbeef;
3122       if (src->type().needsValueReg() ||
3123           RuntimeOption::EvalHHIRGenerateAsserts) {
3124         emitLoadImm(m_as, imm, r);
3125       }
3126     }
3127     if (rd.numAllocated() == 2 && rs.numAllocated() < 2) {
3128       // move a src known type to a dest register
3129       //         a.emitImmReg(arg.imm().q(), dst);
3130       assert(src->type().isKnownDataType());
3131       m_as.emitImmReg(src->type().toDataType(), rd.reg(1));
3132     }
3133   }
3134 }
3135
3136 void CodeGenerator::cgStProp(IRInstruction* inst) {
3137   auto objReg = srcLoc(0).reg();
3138   auto propOff  = inst->src(1)->intVal();
3139   cgStore(objReg[propOff], inst->src(2), srcLoc(2), Width::Full);
3140 }
3141
3142 void CodeGenerator::cgStMem(IRInstruction* inst) {
3143   auto ptr = srcLoc(0).reg();
3144   auto offset = inst->src(1)->intVal();
3145   cgStore(ptr[offset], inst->src(2), srcLoc(2), Width::Full);
3146 }
3147
3148 void CodeGenerator::cgStRef(IRInstruction* inst) {
3149   always_assert(!srcLoc(1).isFullSIMD());
3150   auto destReg = dstLoc(0).reg();
3151   auto ptr = srcLoc(0).reg();
3152   auto off = RefData::tvOffset();
3153   cgStore(ptr[off], inst->src(1), srcLoc(1), Width::Full);
3154   if (destReg != InvalidReg) emitMovRegReg(m_as, ptr, destReg);
3155 }
3156
3157 int CodeGenerator::iterOffset(uint32_t id) {
3158   const Func* func = curFunc();
3159   return -cellsToBytes(((id + 1) * kNumIterCells + func->numLocals()));
3160 }
3161
3162 void CodeGenerator::cgStLoc(IRInstruction* inst) {
3163   auto ptr = srcLoc(0).reg();
3164   auto off = localOffset(inst->extra<StLoc>()->locId);
3165   cgStore(ptr[off], inst->src(1), srcLoc(1), Width::Full);
3166 }
3167
3168 void CodeGenerator::cgStLocNT(IRInstruction* inst) {
3169   auto ptr = srcLoc(0).reg();
3170   auto off = localOffset(inst->extra<StLocNT>()->locId);
3171   cgStore(ptr[off], inst->src(1), srcLoc(1), Width::Value);
3172 }
3173
3174 void CodeGenerator::cgSyncABIRegs(IRInstruction* inst) {
3175   emitMovRegReg(m_as, srcLoc(0).reg(), rVmFp);
3176   emitMovRegReg(m_as, srcLoc(1).reg(), rVmSp);
3177 }
3178
3179 void CodeGenerator::cgEagerSyncVMRegs(IRInstruction* inst) {
3180   always_assert(
3181     srcLoc(0).reg() == rVmFp &&
3182     srcLoc(1).reg() == rVmSp
3183   );
3184   emitEagerSyncPoint(
3185     m_as,
3186     reinterpret_cast<const Op*>(inst->marker().sk().pc())
3187   );
3188 }
3189
3190 void CodeGenerator::cgReqBindJmp(IRInstruction* inst) {
3191   auto offset  = inst->extra<ReqBindJmp>()->offset;
3192   auto trflags = inst->extra<ReqBindJmp>()->trflags;
3193
3194   emitBindJmp(
3195     m_mainCode,
3196     m_frozenCode,
3197     SrcKey(curFunc(), offset, resumed()),
3198     trflags
3199   );
3200 }
3201
3202 void CodeGenerator::cgReqRetranslateOpt(IRInstruction* inst) {
3203   auto extra = inst->extra<ReqRetranslateOpt>();
3204
3205   TCA sr = emitServiceReq(
3206     m_coldCode, REQ_RETRANSLATE_OPT,
3207     SrcKey(curFunc(), extra->offset, resumed()).toAtomicInt(),
3208     extra->transId);
3209   if (m_mainCode.frontier() != m_coldCode.frontier()) {
3210     m_as.jmp(sr);
3211   }
3212 }
3213
3214 void CodeGenerator::cgReqRetranslate(IRInstruction* inst) {
3215   assert(m_unit.bcOff() == inst->marker().bcOff());
3216   auto const destSK = SrcKey(curFunc(), m_unit.bcOff(), resumed());
3217   auto const destSR = mcg->tx().getSrcRec(destSK);
3218
3219   auto trflags = inst->extra<ReqRetranslate>()->trflags;
3220
3221   if (trflags.packed == 0) {
3222     destSR->emitFallbackJump(m_mainCode);
3223   } else {
3224     destSR->emitFallbackJumpCustom(m_mainCode, m_frozenCode, destSK, trflags);
3225   }
3226 }
3227
3228 void CodeGenerator::cgIncRefWork(Type type, SSATmp* src, PhysLoc srcLoc) {
3229   assert(type.maybeCounted());
3230   auto increfMaybeStatic = [&](Asm& a) {
3231     auto base = srcLoc.reg(0);
3232     if (!type.needsStaticBitCheck()) {
3233       emitIncRef(a, base);
3234     } else {
3235       a.cmpl(0, base[FAST_REFCOUNT_OFFSET]);
3236       static_assert(UncountedValue < 0 && StaticValue < 0, "");
3237       ifThen(a, CC_NS, [&](Asm& a) { emitIncRef(a, base); });
3238     }
3239   };
3240
3241   if (type.isKnownDataType()) {
3242     assert(IS_REFCOUNTED_TYPE(type.toDataType()));
3243     increfMaybeStatic(m_as);
3244   } else {
3245     emitCmpTVType(m_as, KindOfRefCountThreshold, srcLoc.reg(1));
3246     ifThen(m_as, CC_NLE, [&](Asm& a) { increfMaybeStatic(a); });
3247   }
3248 }
3249
3250 void CodeGenerator::cgIncRef(IRInstruction* inst) {
3251   SSATmp* src = inst->src(0);
3252   Type type   = src->type();
3253
3254   if (type.notCounted()) return;
3255
3256   cgIncRefWork(type, src, srcLoc(0));
3257 }
3258
3259 void CodeGenerator::cgIncRefCtx(IRInstruction* inst) {
3260   if (inst->src(0)->isA(Type::Obj)) return cgIncRef(inst);
3261
3262   auto const src = srcLoc(0).reg();
3263   auto& a = m_as;
3264
3265   a.    testb  (0x1, rbyte(src));
3266   ifThen(a, CC_Z, [&](Asm& a) {
3267     emitIncRef(a, src);
3268   });
3269 }
3270
3271 void CodeGenerator::cgDecRefStack(IRInstruction* inst) {
3272   cgDecRefMem(inst->typeParam(),
3273               srcLoc(0).reg(),
3274               cellsToBytes(inst->extra<DecRefStack>()->offset));
3275 }
3276
3277 void CodeGenerator::cgDecRefThis(IRInstruction* inst) {
3278   auto fpReg = srcLoc(0).reg();
3279   auto scratchReg = m_rScratch;
3280
3281   // Load AR->m_this into m_rScratch
3282   auto& v = vmain();
3283   v << loadq{fpReg[AROFF(m_this)], scratchReg};
3284
3285   auto decrefIfAvailable = [&](Vout& v) {
3286     // Check if this is available and we're not in a static context instead
3287     v << testbi{1, Vreg{scratchReg}};
3288     ifThen(v, CC_Z, [&](Vout& v) {
3289       cgDecRefStaticType(v,
3290         Type::Obj,
3291         scratchReg,
3292         true /* genZeroCheck */
3293       );
3294     });
3295   };
3296
3297   if (curFunc()->isPseudoMain()) {
3298     // In pseudo-mains, emit check for presence of m_this
3299     v << testq{scratchReg, scratchReg};
3300     ifThen(v, CC_NZ, [&](Vout& v) { decrefIfAvailable(v); });
3301   } else {
3302     decrefIfAvailable(v);
3303   }
3304 }
3305
3306 void CodeGenerator::cgDecRefLoc(IRInstruction* inst) {
3307   cgDecRefMem(inst->typeParam(),
3308               srcLoc(0).reg(),
3309               localOffset(inst->extra<DecRefLoc>()->locId));
3310 }
3311
3312 void CodeGenerator::cgGenericRetDecRefs(IRInstruction* inst) {
3313   auto const rFp       = srcLoc(0).reg();
3314   auto const numLocals = curFunc()->numLocals();
3315   auto& v = vmain();
3316
3317   assert(rFp == rVmFp &&
3318          "free locals helper assumes the frame pointer is rVmFp");
3319
3320   if (numLocals == 0) return;
3321
3322   // The helpers called below use a special ABI, in which r14 and r15 is
3323   // not saved, and the stub expects the stack to be imbalanced (RSP%16==0)
3324   // on entry. So save r14 and r15 in addition to the caller-save registers,
3325   // and use PhysRegSaverStub which assumes the odd stack parity.
3326   auto toSave = m_state.liveRegs[inst] &
3327                 (kCallerSaved | RegSet(r14) | RegSet(r15));
3328   PhysRegSaverStub saver(v, toSave);
3329
3330   auto const target = numLocals > kNumFreeLocalsHelpers
3331     ? mcg->tx().uniqueStubs.freeManyLocalsHelper
3332     : mcg->tx().uniqueStubs.freeLocalsHelpers[numLocals - 1];
3333
3334   v << lea{rFp[-numLocals * sizeof(TypedValue)], r14};
3335   v << call{target};
3336   recordSyncPoint(v);
3337 }
3338
3339 /*
3340  * Depending on the current translation kind, do nothing, profile, or collect
3341  * profiling data for the current DecRef* instruction
3342  *
3343  * Returns true iff the release path for this DecRef should be put in cold
3344  * code.
3345  */
3346 bool CodeGenerator::decRefDestroyIsUnlikely(OptDecRefProfile& profile,
3347                                             Type type) {
3348   auto const kind = mcg->tx().mode();
3349   if (kind != TransKind::Profile && kind != TransKind::Optimize) return true;
3350
3351   // For a profiling key, we use:
3352   // "DecRefProfile-{opcode name}-{stack/local id if present}-{type}"
3353   // This gives good uniqueness within a bytecode without requiring us to track
3354   // more complex things like "this is the 3rd DecRef in this bytecode".
3355   const int32_t profileId =
3356     m_curInst->is(DecRefLoc) ? m_curInst->extra<DecRefLoc>()->locId
3357   : m_curInst->is(DecRefStack) ? m_curInst->extra<DecRefStack>()->offset
3358   : 0;
3359   auto const profileKey =
3360     makeStaticString(folly::to<std::string>("DecRefProfile-",
3361                                             opcodeName(m_curInst->op()),
3362                                             '-',
3363                                             profileId,
3364                                             '-',
3365                                             type.toString()));
3366   profile.emplace(m_unit.context(), m_curInst->marker(), profileKey);
3367
3368   auto& v = vmain();
3369   if (profile->profiling()) {
3370     v << incwm{rVmTl[profile->handle() + offsetof(DecRefProfile, decrement)]};
3371   } else if (profile->optimizing()) {
3372     auto const data = profile->data(DecRefProfile::reduce);
3373     if (data.hitRate() != 0 && data.hitRate() != 100) {
3374       // These are the only interesting cases where we could be doing better.
3375       FTRACE(5, "DecRefProfile: {}: {} {}\n",
3376              data, m_curInst->marker().show(), profileKey->data());
3377     }
3378     if (data.hitRate() == 0) {
3379       v << incstat{Stats::TC_DecRef_Profiled_0};
3380     } else if (data.hitRate() == 100) {
3381       v << incstat{Stats::TC_DecRef_Profiled_100};
3382     }
3383     return data.hitRate() < RuntimeOption::EvalJitUnlikelyDecRefPercent;
3384   }
3385
3386   return true;
3387 }
3388
3389 namespace {
3390 template <typename T>
3391 struct CheckValid {
3392   static bool valid(const T& f) { return true; }
3393 };
3394
3395 template <>
3396 struct CheckValid<void(*)(Asm&)> {
3397   static bool valid(void (*f)(Asm&)) { return f != nullptr; }
3398 };
3399 template <>
3400 struct CheckValid<void(*)(Vout&)> {
3401   static bool valid(void (*f)(Vout&)) { return f != nullptr; }
3402 };
3403 }
3404
3405 //
3406 // Using the given dataReg, this method generates code that checks the static
3407 // bit out of dataReg, and emits a DecRef if needed.
3408 // NOTE: the flags are left with the result of the DecRef's subtraction,
3409 //       which can then be tested immediately after this.
3410 //
3411 // We've tried a variety of tweaks to this and found the current state of
3412 // things optimal, at least when the measurements were made:
3413 // - whether to load the count into a register (if one is available)
3414 // - whether to use if (!--count) release(); if we don't need a static check
3415 // - whether to skip using the register and just emit --count if we know
3416 //   its not static, and can't hit zero.
3417 //
3418 // Return value: the address to be patched if a RefCountedStaticValue check is
3419 //               emitted; NULL otherwise.
3420 //
3421 template <typename F>
3422 void CodeGenerator::cgCheckStaticBitAndDecRef(Vout& v, Vlabel done,
3423                                               Type type, PhysReg dataReg,
3424                                                F destroyImpl) {
3425   always_assert(type.maybeCounted());
3426   bool hasDestroy = CheckValid<F>::valid(destroyImpl);
3427
3428   OptDecRefProfile profile;
3429   auto const unlikelyDestroy =
3430     hasDestroy ? decRefDestroyIsUnlikely(profile, type) : false;
3431
3432   if (hasDestroy) {
3433     v << incstat{unlikelyDestroy ? Stats::TC_DecRef_Normal_Decl :
3434                  Stats::TC_DecRef_Likely_Decl};
3435   } else {
3436     v << incstat{Stats::TC_DecRef_NZ};
3437   }
3438
3439   auto destroy = [&](Vout& v) {
3440     v << incstat{unlikelyDestroy ? Stats::TC_DecRef_Normal_Destroy :
3441                  Stats::TC_DecRef_Likely_Destroy};
3442     if (profile && profile->profiling()) {
3443       v << incwm{rVmTl[profile->handle() + offsetof(DecRefProfile, destroy)]};
3444     }
3445     destroyImpl(v);
3446   };
3447
3448   if (!type.needsStaticBitCheck()) {
3449     v << declm{dataReg[FAST_REFCOUNT_OFFSET]};
3450     if (RuntimeOption::EvalHHIRGenerateAsserts) {
3451       // Assert that the ref count is not less than zero
3452       emitAssertFlagsNonNegative(v);
3453     }
3454
3455     if (hasDestroy) {
3456       ifBlock(v, vcold(), CC_E, destroy, unlikelyDestroy);
3457     }
3458     return;
3459   }
3460
3461   auto static_check_and_decl = [&](Vout& v) {
3462     static_assert(UncountedValue == UNCOUNTED, "");
3463     static_assert(StaticValue == STATIC, "");
3464
3465     if (type.needsStaticBitCheck()) {
3466       auto next = v.makeBlock();
3467       v << jcc{CC_L, {next, done}};
3468       v = next;
3469     }
3470
3471     // Decrement _count
3472     v << declm{dataReg[FAST_REFCOUNT_OFFSET]};
3473     if (RuntimeOption::EvalHHIRGenerateAsserts) {
3474       // Assert that the ref count is not less than zero
3475       emitAssertFlagsNonNegative(v);
3476     }
3477   };
3478
3479   if (hasDestroy) {
3480     v << cmplim{1, dataReg[FAST_REFCOUNT_OFFSET]};
3481     ifThenElse(v, vcold(), CC_E, destroy, static_check_and_decl,
3482                unlikelyDestroy);
3483     return;
3484   }
3485   if (type.needsStaticBitCheck()) {
3486     v << cmplim{0, dataReg[FAST_REFCOUNT_OFFSET]};
3487   }
3488
3489   static_check_and_decl(v);
3490 }
3491
3492 void CodeGenerator::cgCheckStaticBitAndDecRef(Vout& v, Vlabel done,
3493                                               Type type, PhysReg dataReg) {
3494   return cgCheckStaticBitAndDecRef(v, done, type, dataReg,
3495                                    (void (*)(Vout&))nullptr);
3496 }
3497
3498 //
3499 // Returns the address to be patched with the address to jump to in case
3500 // the type is not ref-counted.
3501 //
3502 void CodeGenerator::cgCheckRefCountedType(PhysReg typeReg, Vlabel done) {
3503   auto& v = vmain();
3504   auto next = v.makeBlock();
3505   emitCmpTVType(v, KindOfRefCountThreshold, typeReg);
3506   v << jcc{CC_LE, {next, done}};
3507   v = next;
3508 }
3509
3510 void CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset,
3511                                           Vlabel done) {
3512   auto& v = vmain();
3513   auto next = v.makeBlock();
3514   emitCmpTVType(v, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]);
3515   v << jcc{CC_LE, {next, done}};
3516   v = next;
3517 }
3518
3519 //
3520 // Generates dec-ref of a typed value with statically known type.
3521 //
3522 void CodeGenerator::cgDecRefStaticType(Vout& v, Type type, PhysReg dataReg,
3523                                        bool genZeroCheck) {
3524   assert(type != Type::Cell && type != Type::Gen);
3525   assert(type.isKnownDataType());
3526
3527   if (type.notCounted()) return;
3528
3529   // Check for UncountedValue or StaticValue if needed,
3530   // do the actual DecRef, and leave flags set based on the subtract result,
3531   // which is tested below
3532   auto done = v.makeBlock();
3533   if (genZeroCheck) {
3534     cgCheckStaticBitAndDecRef(v, done, type, dataReg, [&] (Vout& v) {
3535         // Emit the call to release in m_acold
3536         cgCallHelper(v,
3537                      mcg->getDtorCall(type.toDataType()),
3538                      kVoidDest,
3539                      SyncOptions::kSyncPoint,
3540                      argGroup()
3541                      .reg(dataReg));
3542       });
3543   } else {
3544     cgCheckStaticBitAndDecRef(v, done, type, dataReg);
3545   }
3546   if (!v.closed()) v << jmp{done};
3547   v = done;
3548 }
3549
3550 //
3551 // Generates dec-ref of a typed value with dynamic (statically unknown) type,
3552 // when the type is stored in typeReg.
3553 //
3554 void CodeGenerator::cgDecRefDynamicType(PhysReg typeReg,
3555                                         PhysReg dataReg,
3556                                         bool genZeroCheck) {
3557   // Emit check for ref-counted type
3558   auto& v = vmain();
3559   auto done = v.makeBlock();
3560   cgCheckRefCountedType(typeReg, done);
3561
3562   // Emit check for UncountedValue or StaticValue and the actual DecRef
3563   if (genZeroCheck) {
3564       cgCheckStaticBitAndDecRef(v, done, Type::Cell, dataReg, [&] (Vout& v) {
3565           // load destructor fptr
3566           loadDestructorFunc(v, typeReg, m_rScratch);
3567           // Emit call to release in m_acold
3568           cgCallHelper(v,
3569                        CppCall::indirect(m_rScratch),
3570                        kVoidDest,
3571                        SyncOptions::kSyncPoint,
3572                        argGroup()
3573                        .reg(dataReg));
3574         });
3575   } else {
3576     cgCheckStaticBitAndDecRef(v, done, Type::Cell, dataReg);
3577   }
3578   if (!v.closed()) v << jmp{done};
3579   v = done;
3580 }
3581
3582 //
3583 // Generates dec-ref of a typed value with dynamic (statically
3584 // unknown) type, when all we have is the baseReg and offset of
3585 // the typed value. This method assumes that baseReg is not the
3586 // scratch register.
3587 //
3588 void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
3589                                            int64_t offset) {
3590   auto scratchReg = m_rScratch;
3591   assert(baseReg != scratchReg);
3592   auto& v = vmain();
3593   auto done = v.makeBlock();
3594
3595   // Emit check for ref-counted type
3596   cgCheckRefCountedType(baseReg, offset, done);
3597
3598   v << loadq{baseReg[offset + TVOFF(m_data)], scratchReg};
3599
3600   // Emit check for UncountedValue or StaticValue and the actual DecRef
3601   cgCheckStaticBitAndDecRef(v, done, Type::Cell, scratchReg, [&](Vout& v) {
3602     // Emit call to release in m_acold
3603     v << lea{baseReg[offset], scratchReg};
3604     cgCallHelper(v, CppCall::direct(tv_release_generic),
3605                  kVoidDest,
3606                  SyncOptions::kSyncPoint,
3607                  argGroup().reg(scratchReg));
3608   });
3609
3610   if (!v.closed()) v << jmp{done};
3611   v = done;
3612 }
3613
3614 //
3615 // Generates the dec-ref of a typed value in memory address [baseReg + offset].
3616 // This handles cases where type is either static or dynamic.
3617 //
3618 void CodeGenerator::cgDecRefMem(Type type,
3619                                 PhysReg baseReg,
3620                                 int64_t offset) {
3621   if (type.notCounted()) return;
3622
3623   auto scratchReg = m_rScratch;
3624   assert(baseReg != scratchReg);
3625   auto& v = vmain();
3626   if (type.needsReg()) {
3627     // The type is dynamic, but we don't have two registers available
3628     // to load the type and the data.
3629     cgDecRefDynamicTypeMem(baseReg, offset);
3630   } else if (type.maybeCounted()) {
3631     v << loadq{baseReg[offset + TVOFF(m_data)], scratchReg};
3632     cgDecRefStaticType(v, type, scratchReg, true);
3633   }
3634 }
3635
3636 void CodeGenerator::cgDecRefMem(IRInstruction* inst) {
3637   assert(inst->src(0)->type().isPtr());
3638   cgDecRefMem(inst->typeParam(),
3639               srcLoc(0).reg(),
3640               inst->src(1)->intVal());
3641 }
3642
3643 void CodeGenerator::cgDecRefWork(IRInstruction* inst, bool genZeroCheck) {
3644   SSATmp* src   = inst->src(0);
3645   if (!isRefCounted(src)) return;
3646   Type type = src->type();
3647   if (type.isKnownDataType()) {
3648     cgDecRefStaticType(vmain(), type, srcLoc(0).reg(), genZeroCheck);
3649   } else {
3650     cgDecRefDynamicType(srcLoc(0).reg(1), srcLoc(0).reg(0), genZeroCheck);
3651   }
3652 }
3653
3654 void CodeGenerator::cgDecRef(IRInstruction *inst) {
3655   // DecRef may bring the count to zero, and run the destructor.
3656   // Generate code for this.
3657   cgDecRefWork(inst, true);
3658 }
3659
3660 void CodeGenerator::cgDecRefNZ(IRInstruction* inst) {
3661   // DecRefNZ cannot bring the count to zero.
3662   // Therefore, we don't generate zero-checking code.
3663   cgDecRefWork(inst, false);
3664 }
3665
3666 void CodeGenerator::cgCufIterSpillFrame(IRInstruction* inst) {
3667   auto const nArgs = inst->extra<CufIterSpillFrame>()->args;
3668   auto const iterId = inst->extra<CufIterSpillFrame>()->iterId;
3669   auto const itOff = iterOffset(iterId);
3670
3671   const auto spOffset = -safe_cast<int32_t>(kNumActRecCells * sizeof(Cell));
3672   auto spReg = srcLoc(0).reg();
3673   auto fpReg = srcLoc(1).reg();
3674
3675   m_as.loadq   (fpReg[itOff + CufIter::funcOff()], m_rScratch);
3676   m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_func))]);
3677
3678   m_as.loadq   (fpReg[itOff + CufIter::ctxOff()], m_rScratch);
3679   m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_this))]);
3680
3681   m_as.shrq    (1, m_rScratch);
3682   ifThen(m_as, CC_NBE, [this](Asm& a) {
3683       a.shlq(1, m_rScratch);
3684       emitIncRef(a, m_rScratch);
3685     });
3686   m_as.loadq   (fpReg[itOff + CufIter::nameOff()], m_rScratch);
3687   m_as.testq    (m_rScratch, m_rScratch);
3688   ifThen(m_as, CC_NZ, [this](Asm& a) {
3689       a.cmpl(0, m_rScratch[FAST_REFCOUNT_OFFSET]);
3690       static_assert(UncountedValue < 0 && StaticValue < 0, "");
3691       ifThen(a, CC_NS, [&](Asm& a) { emitIncRef(a, m_rScratch); });
3692       a.orq (ActRec::kInvNameBit, m_rScratch);
3693     });
3694   m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_invName))]);
3695   m_as.storel  (safe_cast<int32_t>(nArgs),
3696                 spReg[spOffset + int(AROFF(m_numArgsAndFlags))]);
3697
3698   emitAdjustSp(spReg, dstLoc(0).reg(), spOffset);
3699 }
3700
3701 void CodeGenerator::cgSpillFrame(IRInstruction* inst) {
3702   auto const func      = inst->src(1);
3703   auto const objOrCls  = inst->src(2);
3704   auto const magicName = inst->extra<SpillFrame>()->invName;
3705   auto const nArgs     = inst->extra<SpillFrame>()->numArgs;
3706   auto& a              = m_as;
3707
3708   const auto spOffset = -safe_cast<int32_t>(kNumActRecCells * sizeof(Cell));
3709
3710   auto spReg = srcLoc(0).reg();
3711   // actRec->m_this
3712   if (objOrCls->isA(Type::Cls)) {
3713     // store class
3714     if (objOrCls->isConst()) {
3715       emitImmStoreq(a, uintptr_t(objOrCls->clsVal()) | 1,
3716                     spReg[spOffset + int(AROFF(m_this))]);
3717     } else {
3718       Reg64 clsPtrReg = srcLoc(2/*objOrCls*/).reg();
3719       a.   movq   (clsPtrReg, m_rScratch);
3720       a.   orq    (1, m_rScratch);
3721       a.   storeq (m_rScratch, spReg[spOffset + int(AROFF(m_this))]);
3722     }
3723   } else if (objOrCls->isA(Type::Obj)) {
3724     // store this pointer
3725     a.     storeq (srcLoc(2/*objOrCls*/).reg(),
3726                    spReg[spOffset + int(AROFF(m_this))]);
3727   } else if (objOrCls->isA(Type::Ctx)) {
3728     // Stores either a this pointer or a Cctx -- statically unknown.
3729     Reg64 objOrClsPtrReg = srcLoc(2/*objOrCls*/).reg();
3730     a.     storeq (objOrClsPtrReg, spReg[spOffset + int(AROFF(m_this))]);
3731   } else {
3732     assert(objOrCls->isA(Type::Nullptr));
3733     // no obj or class; this happens in FPushFunc
3734     int offset_m_this = spOffset + int(AROFF(m_this));
3735     a.     storeq (0, spReg[offset_m_this]);
3736   }
3737   // actRec->m_invName
3738   // ActRec::m_invName is encoded as a pointer with bit kInvNameBit
3739   // set to distinguish it from m_varEnv and m_extrArgs
3740   uintptr_t invName = !magicName
3741     ? 0
3742     : reinterpret_cast<uintptr_t>(magicName) | ActRec::kInvNameBit;
3743   emitImmStoreq(a, invName, spReg[spOffset + int(AROFF(m_invName))]);
3744   // actRec->m_func  and possibly actRec->m_cls
3745   // Note m_cls is unioned with m_this and may overwrite previous value
3746   if (func->isA(Type::Nullptr)) {
3747     // No need to store the null---we're always about to run another
3748     // instruction that will populate the Func.
3749   } else if (func->isConst()) {
3750     const Func* f = func->funcVal();
3751     emitImmStoreq(a, intptr_t(f), spReg[spOffset + int(AROFF(m_func))]);
3752   } else {
3753     int offset_m_func = spOffset + int(AROFF(m_func));
3754     auto funcLoc = srcLoc(1);
3755     a.  storeq (funcLoc.reg(0), spReg[offset_m_func]);
3756   }
3757
3758   a.    storel (nArgs, spReg[spOffset + int(AROFF(m_numArgsAndFlags))]);
3759
3760   emitAdjustSp(spReg, dstLoc(0).reg(), spOffset);
3761 }
3762
3763 void CodeGenerator::cgStClosureFunc(IRInstruction* inst) {
3764   auto const obj  = srcLoc(0).reg();
3765   auto const func = inst->extra<StClosureFunc>()->func;
3766   emitImmStoreq(m_as, intptr_t(func), obj[c_Closure::funcOffset()]);
3767 }
3768
3769 void CodeGenerator::cgStClosureArg(IRInstruction* inst) {
3770   cgStore(
3771     srcLoc(0).reg()[inst->extra<StClosureArg>()->offsetBytes],
3772     inst->src(1), srcLoc(1),
3773     Width::Full
3774   );
3775 }
3776
3777 void CodeGenerator::cgStClosureCtx(IRInstruction* inst) {
3778   auto const obj = srcLoc(0).reg();
3779
3780   if (inst->src(1)->isA(Type::Nullptr)) {
3781     m_as.  storeq  (0, obj[c_Closure::ctxOffset()]);
3782   } else {
3783     auto const ctx = srcLoc(1).reg();
3784     always_assert(ctx != InvalidReg);
3785     m_as.  storeq  (ctx, obj[c_Closure::ctxOffset()]);
3786   }
3787 }
3788
3789 void CodeGenerator::emitInitObjProps(PhysReg dstReg,
3790                                      const Class* cls,
3791                                      size_t nProps) {
3792   // If the object has a small number of properties, just emit stores
3793   // inline.
3794   if (nProps < 8) {
3795     for (int i = 0; i < nProps; ++i) {
3796       auto propOffset =
3797         sizeof(ObjectData) + cls->builtinODTailSize() + sizeof(TypedValue) * i;
3798       auto propDataOffset = propOffset + TVOFF(m_data);
3799       auto propTypeOffset = propOffset + TVOFF(m_type);
3800       if (!IS_NULL_TYPE(cls->declPropInit()[i].m_type)) {
3801         emitImmStoreq(m_as, cls->declPropInit()[i].m_data.num,
3802                       dstReg[propDataOffset]);
3803       }
3804       m_as.storeb(cls->declPropInit()[i].m_type, dstReg[propTypeOffset]);
3805     }
3806     return;
3807   }
3808
3809   // Use memcpy for large numbers of properties.
3810   auto args = argGroup()
3811     .addr(dstReg,
3812           safe_cast<int32_t>(sizeof(ObjectData) + cls->builtinODTailSize()))
3813     .imm(int64_t(&cls->declPropInit()[0]))
3814     .imm(cellsToBytes(nProps));
3815   cgCallHelper(m_as,
3816                CppCall::direct(memcpy),
3817                kVoidDest,
3818                SyncOptions::kNoSyncPoint,
3819                args);
3820 }
3821
3822 void CodeGenerator::cgConstructInstance(IRInstruction* inst) {
3823   auto const cls    = inst->extra<ConstructInstance>()->cls;
3824   auto const dstReg = dstLoc(0).reg();
3825   cgCallHelper(m_as,
3826                CppCall::direct(cls->instanceCtor()),
3827                callDest(dstReg),
3828                SyncOptions::kSyncPoint,
3829                argGroup().immPtr(cls));
3830 }
3831
3832 void CodeGenerator::cgCheckInitProps(IRInstruction* inst) {
3833   auto const cls = inst->extra<CheckInitProps>()->cls;
3834   auto const branch = inst->taken();
3835   m_as.cmpq(0, rVmTl[cls->propHandle()]);
3836   emitFwdJcc(m_as, CC_Z, branch);
3837 }
3838
3839 void CodeGenerator::cgCheckInitSProps(IRInstruction* inst) {
3840   auto const cls = inst->extra<CheckInitSProps>()->cls;
3841   auto const branch = inst->taken();
3842   m_as.cmpb(0, rVmTl[cls->sPropInitHandle()]);
3843   emitFwdJcc(m_as, CC_Z, branch);
3844 }
3845
3846 void CodeGenerator::cgNewInstanceRaw(IRInstruction* inst) {
3847   auto const cls    = inst->extra<NewInstanceRaw>()->cls;
3848   auto const dstReg = dstLoc(0).reg();
3849   size_t size = ObjectData::sizeForNProps(cls->numDeclProperties());
3850   cgCallHelper(m_as,
3851                size <= kMaxSmartSize
3852                ? CppCall::direct(ObjectData::newInstanceRaw)
3853                : CppCall::direct(ObjectData::newInstanceRawBig),
3854                callDest(dstReg),
3855                SyncOptions::kSyncPoint,
3856                argGroup().imm((uint64_t)cls).imm(size));
3857 }
3858
3859 void CodeGenerator::cgInitObjProps(IRInstruction* inst) {
3860   auto const cls    = inst->extra<InitObjProps>()->cls;
3861   auto const srcReg = srcLoc(0).reg();
3862
3863   // Set the attributes, if any
3864   int odAttrs = cls->getODAttrs();
3865   if (odAttrs) {
3866     // o_attribute is 16 bits but the fact that we're or-ing a mask makes it ok
3867     assert(!(odAttrs & 0xffff0000));
3868     m_as.orq(odAttrs, srcReg[ObjectData::attributeOff()]);
3869   }
3870
3871   // Initialize the properties
3872   size_t nProps = cls->numDeclProperties();
3873   if (nProps > 0) {
3874     if (cls->pinitVec().size() == 0) {
3875       // Fast case: copy from a known address in the Class
3876       emitInitObjProps(srcReg, cls, nProps);
3877     } else {
3878       // Slower case: we have to load the src address from the targetcache
3879       auto rPropData = m_rScratch;
3880       // Load the Class's propInitVec from the targetcache
3881       m_as.loadq(rVmTl[cls->propHandle()], rPropData);
3882       // propData holds the PropInitVec. We want &(*propData)[0]
3883       m_as.loadq(rPropData[Class::PropInitVec::dataOff()], rPropData);
3884       if (!cls->hasDeepInitProps()) {
3885         auto args = argGroup()
3886           .addr(srcReg,
3887               safe_cast<int32_t>(sizeof(ObjectData) + cls->builtinODTailSize()))
3888           .reg(rPropData)
3889           .imm(cellsToBytes(nProps));
3890         cgCallHelper(m_as,
3891                      CppCall::direct(memcpy),
3892                      kVoidDest,
3893                      SyncOptions::kNoSyncPoint,
3894                      args);
3895       } else {
3896         auto args = argGroup()
3897           .addr(srcReg,
3898               safe_cast<int32_t>(sizeof(ObjectData) + cls->builtinODTailSize()))
3899           .reg(rPropData)
3900           .imm(nProps);
3901         cgCallHelper(m_as,
3902                      CppCall::direct(deepInitHelper),
3903                      kVoidDest,
3904                      SyncOptions::kNoSyncPoint,
3905                      args);
3906       }
3907     }
3908   }
3909 }
3910
3911 void CodeGenerator::cgCallArray(IRInstruction* inst) {
3912   Offset pc             = inst->extra<CallArray>()->pc;
3913   Offset after          = inst->extra<CallArray>()->after;
3914   cgCallHelper(
3915     m_as,
3916     CppCall::direct(
3917       reinterpret_cast<void (*)()>(mcg->tx().uniqueStubs.fcallArrayHelper)),
3918     kVoidDest,
3919     SyncOptions::kSyncPoint,
3920     argGroup()
3921       .imm(pc)
3922       .imm(after)
3923   );
3924 }
3925
3926 void CodeGenerator::cgCall(IRInstruction* inst) {
3927   auto const extra = inst->extra<Call>();
3928   auto const rSP   = srcLoc(0).reg();
3929   auto const rFP   = srcLoc(1).reg();
3930   auto& a = m_as;
3931
3932   auto const ar = extra->numParams * sizeof(TypedValue);
3933   a.    storeq (rFP,                              rSP[ar + AROFF(m_sfp)]);
3934   a.    storel (safe_cast<int32_t>(extra->after), rSP[ar + AROFF(m_soff)]);
3935
3936   if (extra->knownPrologue) {
3937     assert(extra->callee);
3938     if (RuntimeOption::EvalHHIRGenerateAsserts) {
3939       auto const off = cellsToBytes(extra->numParams) + AROFF(m_savedRip);
3940       emitImmStoreq(a, 0xff00ff00b00b00d0, rSP[off]);
3941     }
3942     emitLea(a, rSP[cellsToBytes(extra->numParams)], rStashedAR);
3943     /*
3944      * Normally there's no need to prepare for smash if this is a live
3945      * or optimized translation, since we know where we are going.
3946      *
3947      * However, if we're going to a profiling prologue, we want it to
3948      * be smashable later, so we need to tell the profiling module
3949      * about this and prepare for smashing the call.
3950      */
3951     if (mcg->code.prof().contains(extra->knownPrologue)) {
3952       auto const calleeNumParams = extra->callee->numNonVariadicParams();
3953       auto const prologIndex =
3954         extra->numParams <= calleeNumParams ? extra->numParams
3955                                             : calleeNumParams + 1;
3956       mcg->backEnd().prepareForSmash(a.code(), kCallLen);
3957       mcg->tx().profData()->addPrologueMainCaller(
3958         extra->callee,
3959         prologIndex,
3960         a.code().frontier()
3961       );
3962       always_assert(mcg->backEnd().isSmashable(a.code().frontier(), kCallLen));
3963     }
3964     a.  call   (extra->knownPrologue);
3965     return;
3966   }
3967
3968   assert(dstLoc(0).reg() == rVmSp);
3969   auto const srcKey = m_curInst->marker().sk();
3970   emitBindCall(m_mainCode, m_coldCode, m_frozenCode, srcKey, extra->callee,
3971                extra->numParams);
3972 }
3973
3974 void CodeGenerator::cgCastStk(IRInstruction *inst) {
3975   Type type       = inst->typeParam();
3976   uint32_t offset = inst->extra<CastStk>()->offset;
3977   PhysReg spReg   = srcLoc(0).reg();
3978   auto args = argGroup();
3979   args.addr(spReg, cellsToBytes(offset));
3980
3981   TCA tvCastHelper;
3982   if (type <= Type::Bool) {
3983     tvCastHelper = (TCA)tvCastToBooleanInPlace;
3984   } else if (type <= Type::Int) {
3985     tvCastHelper = (TCA)tvCastToInt64InPlace;
3986   } else if (type <= Type::Dbl) {
3987     tvCastHelper = (TCA)tvCastToDoubleInPlace;
3988   } else if (type <= Type::Arr) {
3989     tvCastHelper = (TCA)tvCastToArrayInPlace;
3990   } else if (type <= Type::Str) {
3991     tvCastHelper = (TCA)tvCastToStringInPlace;
3992   } else if (type <= Type::Obj) {
3993     tvCastHelper = (TCA)tvCastToObjectInPlace;
3994   } else if (type <= Type::NullableObj) {
3995     tvCastHelper = (TCA)tvCastToNullableObjectInPlace;
3996   } else if (type <= Type::Res) {
3997     tvCastHelper = (TCA)tvCastToResourceInPlace;
3998   } else {
3999     not_reached();
4000   }
4001   cgCallHelper(m_as,
4002                CppCall::direct(reinterpret_cast<void (*)()>(tvCastHelper)),
4003                kVoidDest,
4004                SyncOptions::kSyncPoint,
4005                args);
4006 }
4007
4008 void CodeGenerator::cgCastStkIntToDbl(IRInstruction* inst) {
4009   auto spReg = srcLoc(0).reg();
4010   auto offset = cellsToBytes(inst->extra<CastStkIntToDbl>()->offset);
4011   m_as.cvtsi2sd(refTVData(spReg[offset]), rCgXMM0);
4012   emitStoreReg(m_as, rCgXMM0, refTVData(spReg[offset]));
4013   emitStoreTVType(m_as, KindOfDouble, refTVType(spReg[offset]));
4014 }
4015
4016 void CodeGenerator::cgCoerceStk(IRInstruction *inst) {
4017   Type type       = inst->typeParam();
4018   uint32_t offset = inst->extra<CoerceStk>()->offset;
4019   Block* exit     = inst->taken();
4020   PhysReg spReg   = srcLoc(0).reg();
4021
4022   auto args = argGroup();
4023   args.addr(spReg, cellsToBytes(offset));
4024
4025   TCA tvCoerceHelper;
4026   if (type <= Type::Bool) {
4027     tvCoerceHelper = (TCA)tvCoerceParamToBooleanInPlace;
4028   } else if (type <= Type::Int) {
4029     // if casting to integer, pass 10 as the base for the conversion
4030     args.imm(10);
4031     tvCoerceHelper = (TCA)tvCoerceParamToInt64InPlace;
4032   } else if (type <= Type::Dbl) {
4033     tvCoerceHelper = (TCA)tvCoerceParamToDoubleInPlace;
4034   } else if (type <= Type::Arr) {
4035     tvCoerceHelper = (TCA)tvCoerceParamToArrayInPlace;
4036   } else if (type <= Type::Str) {
4037     tvCoerceHelper = (TCA)tvCoerceParamToStringInPlace;
4038   } else if (type <= Type::Obj) {
4039     tvCoerceHelper = (TCA)tvCoerceParamToObjectInPlace;
4040   } else if (type <= Type::Res) {
4041     tvCoerceHelper = (TCA)tvCoerceParamToResourceInPlace;
4042   } else {
4043     not_reached();
4044   }
4045
4046   auto tmpReg = PhysReg(m_rScratch);
4047   cgCallHelper(m_as,
4048     CppCall::direct(reinterpret_cast<void (*)()>(tvCoerceHelper)),
4049     callDest(tmpReg),
4050     SyncOptions::kSyncPoint,
4051     args
4052   );
4053   m_as.testb(1, rbyte(tmpReg));
4054   emitFwdJcc(m_as, CC_E, exit);
4055 }
4056
4057 void CodeGenerator::cgCallBuiltin(IRInstruction* inst) {
4058   auto const dst            = dstLoc(0);
4059   auto const dstReg         = dst.reg(0);
4060   auto const dstType        = dst.reg(1);
4061   auto const callee         = inst->extra<CallBuiltin>()->callee;
4062   auto const numArgs        = callee->numParams();
4063   auto const returnType     = inst->typeParam();
4064   auto const funcReturnType = callee->returnType();
4065
4066   int returnOffset = MISOFF(tvBuiltinReturn);
4067
4068   if (FixupMap::eagerRecord(callee)) {
4069     auto const pc = curUnit()->entry() + m_curInst->marker().bcOff();
4070     // we have spilled all args to stack, so spDiff is 0
4071     emitEagerSyncPoint(m_as, reinterpret_cast<const Op*>(pc));
4072   }
4073   // RSP points to the MInstrState we need to use.  Workaround the
4074   // fact that rsp moves when we spill registers around call
4075   PhysReg misReg = m_rScratch;
4076   emitMovRegReg(m_as, reg::rsp, misReg);
4077
4078   auto callArgs = argGroup();
4079   if (isCppByRef(funcReturnType)) {
4080     // First arg is pointer to storage for that return value
4081     if (isSmartPtrRef(funcReturnType)) {
4082       returnOffset += TVOFF(m_data);
4083     }
4084     // misReg is pointing to an MInstrState struct on the C stack.  Pass
4085     // the address of tvBuiltinReturn to the native function as the location
4086     // it can construct the return Array, String, Object, or Variant.
4087     callArgs.addr(misReg, returnOffset); // &misReg[returnOffset]
4088   }
4089
4090   // Non-pointer args are plain values passed by value.  String, Array,
4091   // Object, and Variant are passed by const&, ie a pointer to stack memory
4092   // holding the value, so expect PtrToT types for these.
4093   // Pointers to smartptr types (String, Array, Object) need adjusting to
4094   // point to &ptr->m_data.
4095   auto srcNum = uint32_t{0};
4096   if (callee->isMethod() && !(callee->attrs() & AttrStatic)) {
4097     // Note, we don't support objects with vtables here (if they may
4098     // need a this pointer adjustment).  This should be filtered out
4099     // earlier right now.
4100     callArgs.ssa(srcNum);
4101     ++srcNum;
4102   }
4103   for (uint32_t i = 0; i < numArgs; ++i, ++srcNum) {
4104     auto const& pi = callee->params()[i];
4105     if (TVOFF(m_data) && isSmartPtrRef(pi.builtinType)) {
4106       assert(inst->src(srcNum)->type().isPtr() &&
4107              srcLoc(srcNum).reg() != InvalidReg);
4108       callArgs.addr(srcLoc(srcNum).reg(), TVOFF(m_data));
4109     } else {
4110       callArgs.ssa(srcNum, pi.builtinType == KindOfDouble);
4111     }
4112   }
4113
4114   // If the return value is returned by reference, we don't need the
4115   // return value from this call since we know where the value is.
4116   auto dest = isCppByRef(funcReturnType) ? kVoidDest :
4117               funcReturnType == KindOfDouble ? callDestDbl(inst) :
4118               callDest(inst);
4119   cgCallHelper(m_as, CppCall::direct(callee->nativeFuncPtr()),
4120                dest, SyncOptions::kSyncPoint, callArgs);
4121
4122   // For primitive return types (int, bool, double), the return value
4123   // is already in dstReg (the builtin call returns in rax or xmm0).
4124   // For void return types, there is no return value, so just exit.
4125   if (dstReg == InvalidReg || returnType.isSimpleType()) {
4126     return;
4127   }
4128
4129   // after the call, RSP is back pointing to MInstrState and rSratch
4130   // has been clobberred.
4131   misReg = rsp;
4132
4133   // For return by reference (String, Object, Array, Variant),
4134   // the builtin writes the return value into MInstrState::tvBuiltinReturn
4135   // TV, from where it has to be tested and copied.
4136   if (returnType.isReferenceType()) {
4137     assert(isCppByRef(funcReturnType) && isSmartPtrRef(funcReturnType));
4138     // return type is String, Array, or Object; fold nullptr to KindOfNull
4139     m_as.   loadq (misReg[returnOffset], dstReg);
4140     emitLoadImm(m_as, returnType.toDataType(), dstType);
4141     emitLoadImm(m_as, KindOfNull, m_rScratch);
4142     m_as.   testq (dstReg, dstReg);
4143     m_as.   cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
4144     return;
4145   }
4146   if (returnType <= Type::Cell || returnType <= Type::BoxedCell) {
4147     // return type is Variant; fold KindOfUninit to KindOfNull
4148     assert(isCppByRef(funcReturnType) && !isSmartPtrRef(funcReturnType));
4149     assert(misReg != dstType);
4150     emitLoadTVType(m_as, misReg[returnOffset + TVOFF(m_type)], dstType);
4151     m_as.   loadq (misReg[returnOffset + TVOFF(m_data)], dstReg);
4152     emitLoadImm(m_as, KindOfNull, m_rScratch);
4153     static_assert(KindOfUninit == 0, "KindOfUninit must be 0 for test");
4154     m_as.   testb (rbyte(dstType), rbyte(dstType));
4155     m_as.   cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
4156     return;
4157   }
4158   not_reached();
4159 }
4160
4161 void CodeGenerator::cgSpillStack(IRInstruction* inst) {
4162   auto const spDeficit    = inst->src(1)->intVal();
4163   auto const spillVals    = inst->srcs().subpiece(2);
4164   auto const numSpillSrcs = spillVals.size();
4165   auto const dstReg       = dstLoc(0).reg();
4166   auto const spReg        = srcLoc(0).reg();
4167   auto const spillCells   = spillValueCells(inst);
4168
4169   int adjustment = safe_cast<int32_t>(
4170     (spDeficit - spillCells) * ssize_t(sizeof(Cell))
4171   );
4172   for (uint32_t i = 0; i < numSpillSrcs; ++i) {
4173     int offset = safe_cast<int32_t>(i * ssize_t(sizeof(Cell)) + adjustment);
4174     cgStore(spReg[offset], spillVals[i], srcLoc(i + 2), Width::Full);
4175   }
4176   emitAdjustSp(spReg, dstReg, adjustment);
4177 }
4178
4179 void CodeGenerator::emitAdjustSp(PhysReg spReg, PhysReg dstReg,
4180                                  int adjustment /* bytes */) {
4181   if (adjustment != 0) {
4182     if (dstReg != spReg) {
4183       m_as.    lea   (spReg[adjustment], dstReg);
4184     } else {
4185       m_as.    addq  (adjustment, dstReg);
4186     }
4187   } else {
4188     emitMovRegReg(m_as, spReg, dstReg);
4189   }
4190 }
4191
4192 void CodeGenerator::cgNativeImpl(IRInstruction* inst) {
4193   auto const func = curFunc();
4194   auto const builtinFuncPtr = func->builtinFuncPtr();
4195   auto& v = vmain();
4196
4197   v << copy{srcLoc(0).reg(), argNumToRegName[0]};
4198   if (FixupMap::eagerRecord(func)) {
4199     emitEagerSyncPoint(v, reinterpret_cast<const Op*>(func->getEntry()));
4200   }
4201   v << call{(TCA)builtinFuncPtr};
4202   recordSyncPoint(v);
4203 }
4204
4205 void CodeGenerator::cgLdThis(IRInstruction* inst) {
4206   Block* label  = inst->taken();
4207   auto dstReg = dstLoc(0).reg();
4208
4209   m_as.loadq(srcLoc(0).reg()[AROFF(m_this)], dstReg);
4210   if (label == NULL) return;  // no need to perform its checks
4211
4212   if (curFunc()->isPseudoMain() || !curFunc()->mayHaveThis()) {
4213     // Check for a null $this pointer first.
4214     m_as.testq(dstReg, dstReg);
4215     emitFwdJcc(CC_Z, label);
4216   }
4217
4218   m_as.testb(1, rbyte(dstReg));
4219   emitFwdJcc(CC_NZ, label);
4220 }
4221
4222 void CodeGenerator::cgLdClsCtx(IRInstruction* inst) {
4223   PhysReg srcReg = srcLoc(0).reg();
4224   PhysReg dstReg = dstLoc(0).reg();
4225   // Context could be either a this object or a class ptr
4226   m_as.   testb(1, rbyte(srcReg));
4227   ifThenElse(CC_NZ,
4228              [&](Asm& a) { emitLdClsCctx(a, srcReg, dstReg);  }, // ctx is class
4229              [&](Asm& a) { emitLdObjClass(a, srcReg, dstReg); }  // ctx is $this
4230             );
4231 }
4232
4233 void CodeGenerator::cgLdClsCctx(IRInstruction* inst) {
4234   PhysReg srcReg = srcLoc(0).reg();
4235   PhysReg dstReg = dstLoc(0).reg();
4236   emitLdClsCctx(m_as, srcReg, dstReg);
4237 }
4238
4239 void CodeGenerator::cgLdCtx(IRInstruction* inst) {
4240   auto const dstReg = dstLoc(0).reg();
4241   auto const srcReg = srcLoc(0).reg();
4242   m_as.loadq(srcReg[AROFF(m_this)], dstReg);
4243 }
4244
4245 void CodeGenerator::cgLdCctx(IRInstruction* inst) {
4246   return cgLdCtx(inst);
4247 }
4248
4249 void CodeGenerator::cgLdClsName(IRInstruction* inst) {
4250   auto const dstReg = dstLoc(0).reg();
4251   auto const srcReg = srcLoc(0).reg();
4252
4253   m_as.loadq(srcReg[Class::preClassOff()], dstReg);
4254   emitLdLowPtr(m_as, dstReg[PreClass::nameOffset()],
4255                dstReg, sizeof(LowStringPtr));
4256 }
4257
4258 void CodeGenerator::cgLdARFuncPtr(IRInstruction* inst) {
4259   assert(inst->src(1)->isConst());
4260   auto const offset = inst->src(1);
4261   auto dstReg       = dstLoc(0).reg();
4262   auto baseReg      = srcLoc(0).reg();
4263   m_as.loadq(baseReg[offset->intVal() + AROFF(m_func)], dstReg);
4264 }
4265
4266 void CodeGenerator::cgLdStaticLocCached(IRInstruction* inst) {
4267   auto const extra = inst->extra<LdStaticLocCached>();
4268   auto const link  = RDS::bindStaticLocal(extra->func, extra->name);
4269   auto const dst   = dstLoc(0).reg();
4270   auto& a = m_as;
4271   emitLea(a, rVmTl[link.handle()], dst);
4272 }
4273
4274 void CodeGenerator::cgCheckStaticLocInit(IRInstruction* inst) {
4275   auto const src = srcLoc(0).reg();
4276   auto& a = m_as;
4277   emitCmpTVType(a, KindOfUninit, src[RefData::tvOffset() + TVOFF(m_type)]);
4278   emitFwdJcc(a, CC_E, inst->taken());
4279 }
4280
4281 void CodeGenerator::cgStaticLocInitCached(IRInstruction* inst) {
4282   auto const rdSrc = srcLoc(0).reg();
4283   auto& a = m_as;
4284
4285   // If we're here, the target-cache-local RefData is all zeros, so we
4286   // can initialize it by storing the new value into it's TypedValue
4287   // and incrementing the RefData reference count (which will set it
4288   // to 1).
4289   //
4290   // We are storing the rdSrc value into the static, but we don't need
4291   // to inc ref it because it's a bytecode invariant that it's not a
4292   // reference counted type.
4293   cgStore(rdSrc[RefData::tvOffset()], inst->src(1), srcLoc(1), Width::Full);
4294   a.    incl   (rdSrc[FAST_REFCOUNT_OFFSET]);
4295   if (debug) {
4296     static_assert(sizeof(RefData::Magic::kMagic) == sizeof(uint64_t), "");
4297     emitImmStoreq(a, static_cast<int64_t>(RefData::Magic::kMagic),
4298                   rdSrc[RefData::magicOffset()]);
4299   }
4300 }
4301
4302 void CodeGenerator::cgStoreTypedValue(MemoryRef dst, SSATmp* src, PhysLoc loc) {
4303   assert(src->type().needsReg());
4304   auto srcReg0 = loc.reg(0);
4305   auto srcReg1 = loc.reg(1);
4306   if (srcReg0.isSIMD()) {
4307     // Whole typed value is stored in single SIMD reg srcReg0
4308     assert(RuntimeOption::EvalHHIRAllocSIMDRegs);
4309     assert(srcReg1 == InvalidReg);
4310     m_as.movdqu(srcReg0, refTVData(dst));
4311     return;
4312   }
4313
4314   if (src->type().needsValueReg()) {
4315     assert(srcReg0 != InvalidReg);
4316     m_as.storeq(srcReg0, refTVData(dst));
4317   }
4318
4319   assert(srcReg1 != InvalidReg);
4320   emitStoreTVType(m_as, srcReg1, refTVType(dst));
4321 }
4322
4323 void CodeGenerator::cgStore(MemoryRef dst, SSATmp* src, PhysLoc srcLoc,
4324                             Width width) {
4325   Type type = src->type();
4326   if (type.needsReg()) {
4327     always_assert(width == Width::Full);
4328     cgStoreTypedValue(dst, src, srcLoc);
4329     return;
4330   }
4331   if (width == Width::Full) {
4332     emitStoreTVType(m_as, type.toDataType(), refTVType(dst));
4333   }
4334   if (!src->type().needsValueReg()) return; // no value to store
4335
4336   auto memRef = refTVData(dst);
4337   auto srcReg = srcLoc.reg();
4338   if (srcReg == InvalidReg) {
4339     always_assert(type <= (Type::Bool | Type::Int | Type::Dbl |
4340                   Type::Arr | Type::StaticStr | Type::Cls));
4341     emitImmStoreq(m_as, src->rawVal(), memRef);
4342   } else {
4343     zeroExtendIfBool(m_as, src, srcReg);
4344     emitStoreReg(m_as, srcReg, memRef);
4345   }
4346 }
4347
4348 void CodeGenerator::cgLoad(SSATmp* dst, PhysLoc dstLoc, MemoryRef base,
4349                            Block* label) {
4350   Type type = dst->type();
4351   if (type.needsReg()) {
4352     return cgLoadTypedValue(dst, dstLoc, base, label);
4353   }
4354   if (label != NULL) {
4355     emitTypeCheck(type, refTVType(base), refTVData(base), label);
4356   }
4357   auto dstReg = dstLoc.reg();
4358   // if dstReg == InvalidReg then there's nothing to load.
4359   if (dstReg == InvalidReg) return;
4360   if (type <= Type::Bool) {
4361     m_as.loadl(refTVData(base), toReg32(dstReg));
4362   } else {
4363     emitLoadReg(m_as, refTVData(base), dstReg);
4364   }
4365 }
4366
4367 static MemoryRef makeMemoryRef(Asm& as, Reg64 scratch,
4368                                MemoryRef value) {
4369   always_assert(int(value.r.index) != -1);
4370   always_assert(value.r.scale == 1); //TASK(2731486): fix this... use imul?
4371   if (value.r.base != scratch && value.r.index != scratch) {
4372     as.movq(value.r.base, scratch);
4373     value.r.base = scratch;
4374   }
4375   if (value.r.base == scratch) {
4376     as.addq(value.r.index, value.r.base);
4377     return value.r.base[value.r.disp];
4378   }
4379   as.addq(value.r.base, value.r.index);
4380   return value.r.index[value.r.disp];
4381 }
4382
4383 // If label is not null and type is not Gen, this method generates a check
4384 // that bails to the label if the loaded typed value doesn't match dst type.
4385 void CodeGenerator::cgLoadTypedValue(SSATmp* dst, PhysLoc dstLoc,
4386                                      MemoryRef ref, Block* label) {
4387   Type type = dst->type();
4388   auto valueDstReg = dstLoc.reg(0);
4389   auto typeDstReg  = dstLoc.reg(1);
4390
4391   if (valueDstReg.isSIMD()) {
4392     assert(!label);
4393     // Whole typed value is stored in single SIMD reg valueDstReg
4394     assert(RuntimeOption::EvalHHIRAllocSIMDRegs);
4395     assert(typeDstReg == InvalidReg);
4396     m_as.movdqu(refTVData(ref), valueDstReg);
4397     return;
4398   }
4399
4400   if (valueDstReg == InvalidReg && typeDstReg == InvalidReg &&
4401       (label == nullptr || type == Type::Gen)) {
4402     // a dead load
4403     return;
4404   }
4405
4406   bool isResolved = true;
4407   auto origRef = ref;
4408   ref = resolveRegCollision(typeDstReg, ref, isResolved);
4409   if (!isResolved) {
4410     // there was a collision with the registers that could not be resolved.
4411     // Re-enter with a non-indexed MemoryRef (slow path).
4412     cgLoadTypedValue(dst, dstLoc, makeMemoryRef(m_as, m_rScratch, origRef),
4413                      label);
4414     return;
4415   }
4416   // Load type if it's not dead
4417   if (typeDstReg != InvalidReg) {
4418     emitLoadTVType(m_as, refTVType(ref), typeDstReg);
4419     if (label) {
4420       emitTypeCheck(type, typeDstReg,
4421                     valueDstReg, label);
4422     }
4423   } else if (label) {
4424     emitTypeCheck(type, refTVType(ref), refTVData(ref), label);
4425   }
4426
4427   // Load value if it's not dead
4428   if (valueDstReg == InvalidReg) return;
4429
4430   m_as.loadq(refTVData(ref), valueDstReg);
4431 }
4432
4433 // May set isResolved = false to signal that there is no register conflict
4434 // resolution. Callers should take proper action to solve the issue (e.g.
4435 // change the indexed MemoryRef into a base+disp MemoryRef). In this case,
4436 // the returned MemoryRef should be ignored.
4437 MemoryRef CodeGenerator::resolveRegCollision(PhysReg dst, MemoryRef memRef,
4438                                              bool& isResolved) {
4439   isResolved = true;
4440   Reg64 base = memRef.r.base;
4441   Reg64 index = memRef.r.index;
4442   if (int(index) == -1) {
4443     if (base == dst) {
4444       assert(base != m_rScratch);
4445       // use the scratch register instead
4446       m_as.movq(base, m_rScratch);
4447       return m_rScratch[memRef.r.disp];
4448     }
4449     return memRef;
4450   }
4451   bool scratchTaken = (base == m_rScratch || index == m_rScratch);
4452   if (base == dst) {
4453     if (scratchTaken) {
4454       // bail, the caller will manage somehow
4455       isResolved = false;
4456       return memRef;
4457     }
4458     // use the scratch register instead
4459     m_as.movq(base, m_rScratch);
4460     return m_rScratch[index + memRef.r.disp];
4461   } else if (index == dst) {
4462     if (scratchTaken) {
4463       // bail, the caller will manage somehow
4464       isResolved = false;
4465       return memRef;
4466     }
4467     // use the scratch register instead
4468     m_as.movq(index, m_rScratch);
4469     return base[m_rScratch + memRef.r.disp];
4470   }
4471   return memRef;
4472 }
4473
4474 void CodeGenerator::cgLdProp(IRInstruction* inst) {
4475   cgLoad(inst->dst(), dstLoc(0),
4476          srcLoc(0).reg()[inst->src(1)->intVal()],
4477          inst->taken());
4478 }
4479
4480 void CodeGenerator::cgLdMem(IRInstruction * inst) {
4481   cgLoad(inst->dst(), dstLoc(0),
4482          srcLoc(0).reg()[inst->src(1)->intVal()],
4483          inst->taken());
4484 }
4485
4486 void CodeGenerator::cgLdRef(IRInstruction* inst) {
4487   cgLoad(inst->dst(), dstLoc(0),
4488          srcLoc(0).reg()[RefData::tvOffset()],
4489          inst->taken());
4490 }
4491
4492 void CodeGenerator::cgStringIsset(IRInstruction* inst) {
4493   auto strReg = srcLoc(0).reg();
4494   auto idxReg = srcLoc(1).reg();
4495   auto dstReg = dstLoc(0).reg();
4496   if (idxReg == InvalidReg) {
4497     m_as.cmpl(safe_cast<int32_t>(inst->src(1)->intVal()),
4498               strReg[StringData::sizeOff()]);
4499   } else {
4500     m_as.cmpl(r32(idxReg), strReg[StringData::sizeOff()]);
4501   }
4502   m_as.setnbe(rbyte(dstReg));
4503 }
4504
4505 void CodeGenerator::cgProfileArray(IRInstruction* inst) {
4506   auto baseReg = srcLoc(0).reg();
4507   auto handle  = inst->extra<ProfileArray>()->handle;
4508
4509   // If kPackedKind changes to a value that is not 0, change
4510   // this to a conditional add.
4511   static_assert(ArrayData::ArrayKind::kPackedKind == 0, "kPackedKind changed");
4512   m_as.  loadzbl(baseReg[ArrayData::offsetofKind()], r32(m_rScratch));
4513   m_as.  addl(r32(m_rScratch),
4514               rVmTl[handle + offsetof(NonPackedArrayProfile, count)]);
4515 }
4516
4517 void CodeGenerator::cgCheckPackedArrayBounds(IRInstruction* inst) {
4518   static_assert(ArrayData::sizeofSize() == 4, "");
4519   // We may check packed array bounds on profiled arrays for which
4520   // we do not statically know that they are of kPackedKind.
4521   assert(inst->taken());
4522   auto arrReg = srcLoc(0).reg();
4523   auto idxReg = srcLoc(1).reg();
4524   if (idxReg == InvalidReg) {
4525     m_as.cmpl  (safe_cast<int32_t>(inst->src(1)->intVal()),
4526                 arrReg[ArrayData::offsetofSize()]);
4527   } else {
4528     // ArrayData::m_size is a uint32_t but we need to do a 64-bit comparison
4529     // since idx is KindOfInt64.
4530     m_as.loadl (arrReg[ArrayData::offsetofSize()], r32(m_rScratch));
4531     m_as.cmpq  (idxReg, m_rScratch);
4532   }
4533   emitFwdJcc(CC_BE, inst->taken());
4534 }
4535
4536 void CodeGenerator::cgLdPackedArrayElem(IRInstruction* inst) {
4537   if (inst->src(0)->isConst()) {
4538     // This would require two scratch registers and should be very
4539     // rare. t3626251
4540     CG_PUNT(LdPackedArrayElem-ConstArray);
4541   }
4542
4543   auto const rArr = srcLoc(0).reg();
4544   auto const rIdx = srcLoc(1).reg();
4545   auto& a = m_as;
4546
4547   // We don't know if we have the last use of rIdx, so we can't
4548   // clobber it.
4549   if (rIdx != InvalidReg) {
4550     /*
4551      * gcc 4.8 did something more like:
4552      *
4553      *    lea 1(%base), %scratch   ; sizeof(ArrayData) == sizeof(TypedValue)
4554      *    salq $4, %scratch
4555      *    movq (%base,%scratch,1), %r1
4556      *    movzxb 8(%base,%scratch,1), %r2
4557      *
4558      * Using this way for now (which is more like what clang produced)
4559      * just because it was 2 bytes smaller.
4560      */
4561     static_assert(sizeof(TypedValue) == 16, "");
4562     a.    movq  (rIdx, m_rScratch);
4563     a.    shlq  (0x4, m_rScratch);
4564     cgLoad(inst->dst(), dstLoc(0), rArr[m_rScratch + sizeof(ArrayData)]);
4565     return;
4566   }
4567
4568   auto const idx    = inst->src(1)->intVal();
4569   auto const offset = sizeof(ArrayData) + idx * sizeof(TypedValue);
4570   cgLoad(inst->dst(), dstLoc(0), rArr[offset]);
4571 }
4572
4573 void CodeGenerator::cgCheckPackedArrayElemNull(IRInstruction* inst) {
4574   if (inst->src(0)->isConst()) {
4575     // This would require two scratch registers and should be very
4576     // rare. t3626251
4577     CG_PUNT(LdPackedArrayElemAddr-ConstArray);
4578   }
4579
4580   auto const rArr = srcLoc(0).reg();
4581   auto const rIdx = srcLoc(1).reg();
4582   auto& a = m_as;
4583
4584   if (rIdx != InvalidReg) {
4585     static_assert(sizeof(TypedValue) == 16, "");
4586     a.    movq  (rIdx, m_rScratch);
4587     a.    shlq  (0x4, m_rScratch);
4588     emitCmpTVType(a, KindOfNull,
4589       rArr[m_rScratch + sizeof(ArrayData) + TVOFF(m_type)]);
4590   } else {
4591     auto const idx    = inst->src(1)->intVal();
4592     auto const offset = sizeof(ArrayData) + idx * sizeof(TypedValue);
4593     emitCmpTVType(a, KindOfNull, rArr[offset + TVOFF(m_type)]);
4594   }
4595
4596   auto const dst = dstLoc(0).reg();
4597   a.    setne  (rbyte(dst));
4598   a.    movzbl (rbyte(dst), r32(dst));
4599 }
4600
4601 void CodeGenerator::cgCheckBounds(IRInstruction* inst) {
4602   auto idx = inst->src(0);
4603   auto idxReg = srcLoc(0).reg();
4604   auto size = inst->src(1);
4605   auto sizeReg = srcLoc(1).reg();
4606   // caller made the check if both sources are constant and never
4607   // generate this opcode
4608   assert(!(idx->isConst() && size->isConst()));
4609   auto throwHelper =
4610     [&](Asm& as) {
4611       auto args = argGroup();
4612       args.ssa(0/*idx*/);
4613       cgCallHelper(as, CppCall::direct(throwOOB),
4614                    kVoidDest, SyncOptions::kSyncPoint, args);
4615
4616     };
4617
4618   if (idxReg == InvalidReg) {
4619     assert(idx->intVal() >= 0); // we would have punted otherwise
4620     auto idxVal = safe_cast<int32_t>(idx->intVal());
4621     m_as.cmpq(idxVal, sizeReg);
4622     unlikelyIfBlock(CC_LE, throwHelper);
4623     return;
4624   }
4625
4626   if (sizeReg == InvalidReg) {
4627     assert(size->intVal() >= 0);
4628     auto sizeVal = safe_cast<int32_t>(size->intVal());
4629     m_as.cmpq(sizeVal, idxReg);
4630   } else {
4631     m_as.cmpq(sizeReg, idxReg);
4632   }
4633   unlikelyIfBlock(CC_AE, throwHelper);
4634 }
4635
4636 void CodeGenerator::cgLdVectorSize(IRInstruction* inst) {
4637   DEBUG_ONLY auto vec = inst->src(0);
4638   auto vecReg = srcLoc(0).reg();
4639   auto dstReg = dstLoc(0).reg();
4640   assert(vec->type().strictSubtypeOf(Type::Obj) &&
4641          vec->type().getClass() == c_Vector::classof());
4642   m_as.loadl(vecReg[c_Vector::sizeOffset()], toReg32(dstReg));
4643 }
4644
4645 void CodeGenerator::cgLdVectorBase(IRInstruction* inst) {
4646   DEBUG_ONLY auto vec = inst->src(0);
4647   auto vecReg = srcLoc(0).reg();
4648   auto dstReg = dstLoc(0).reg();
4649   assert(vec->type().strictSubtypeOf(Type::Obj) &&
4650          vec->type().getClass() == c_Vector::classof());
4651   m_as.loadq(vecReg[c_Vector::dataOffset()], dstReg);
4652 }
4653
4654 /**
4655  * Given a vector, check if it has a immutable copy and jump to the taken
4656  * branch if so.
4657  */
4658 void CodeGenerator::cgVectorHasImmCopy(IRInstruction* inst) {
4659   DEBUG_ONLY auto vec = inst->src(0);
4660   auto vecReg = srcLoc(0).reg();
4661
4662   assert(vec->type().strictSubtypeOf(Type::Obj) &&
4663          vec->type().getClass() == c_Vector::classof());
4664
4665   // Vector::m_data field holds an address of an ArrayData plus
4666   // sizeof(ArrayData) bytes. We need to check this ArrayData's
4667   // m_count field to see if we need to call Vector::triggerCow().
4668   uint rawPtrOffset = c_Vector::dataOffset() + kExpectedMPxOffset;
4669
4670   m_as.loadq(vecReg[rawPtrOffset], m_rScratch);
4671   m_as.cmpl(
4672     1,
4673     m_rScratch[(int64_t)FAST_REFCOUNT_OFFSET - (int64_t)sizeof(ArrayData)]
4674   );
4675   emitFwdJcc(CC_NE, inst->taken());
4676 }
4677
4678 /**
4679  * Given the base of a vector object, pass it to a helper
4680  * which is responsible for triggering COW.
4681  */
4682 void CodeGenerator::cgVectorDoCow(IRInstruction* inst) {
4683   DEBUG_ONLY auto vec = inst->src(0);
4684   assert(vec->type().strictSubtypeOf(Type::Obj) &&
4685          vec->type().getClass() == c_Vector::classof());
4686   auto args = argGroup();
4687   args.ssa(0); // vec
4688   cgCallHelper(m_as, CppCall::direct(triggerCow),
4689                kVoidDest, SyncOptions::kSyncPoint, args);
4690 }
4691
4692 void CodeGenerator::cgLdPairBase(IRInstruction* inst) {
4693   DEBUG_ONLY auto pair = inst->src(0);
4694   auto pairReg = srcLoc(0).reg();
4695   assert(pair->type().strictSubtypeOf(Type::Obj) &&
4696          pair->type().getClass() == c_Pair::classof());
4697   m_as.lea(pairReg[c_Pair::dataOffset()], dstLoc(0).reg());
4698 }
4699
4700 void CodeGenerator::cgLdElem(IRInstruction* inst) {
4701   auto baseReg = srcLoc(0).reg();
4702   auto idx = inst->src(1);
4703   auto idxReg = srcLoc(1).reg();
4704   if (idx->isConst()) {
4705     cgLoad(inst->dst(), dstLoc(0), baseReg[idx->intVal()]);
4706   } else {
4707     cgLoad(inst->dst(), dstLoc(0), baseReg[idxReg]);
4708   }
4709 }
4710
4711 void CodeGenerator::cgStElem(IRInstruction* inst) {
4712   auto baseReg = srcLoc(0).reg();
4713   auto srcValue = inst->src(2);
4714   auto idx = inst->src(1);
4715   auto idxReg = srcLoc(1).reg();
4716   if (idxReg == InvalidReg) {
4717     cgStore(baseReg[idx->intVal()], srcValue, srcLoc(2), Width::Full);
4718   } else {
4719     cgStore(baseReg[idxReg], srcValue, srcLoc(2), Width::Full);
4720   }
4721 }
4722
4723 void CodeGenerator::recordSyncPoint(Vout& v,
4724                                     SyncOptions sync /* = kSyncPoint */) {
4725   auto const marker = m_curInst->marker();
4726   assert(m_curInst->marker().valid());
4727
4728   Offset stackOff = marker.spOff();
4729   switch (sync) {
4730   case SyncOptions::kSyncPointAdjustOne:
4731     stackOff -= 1;
4732     break;
4733   case SyncOptions::kSyncPoint:
4734   case SyncOptions::kSmashableAndSyncPoint:
4735     break;
4736   case SyncOptions::kNoSyncPoint:
4737     // we can get here if we are memory profiling, since we override the
4738     // normal sync settings and sync anyway
4739     assert(RuntimeOption::HHProfServerEnabled);
4740   }
4741
4742   Offset pcOff = marker.bcOff() - marker.func()->base();
4743   v << syncpoint{Fixup{pcOff, stackOff}};
4744 }
4745
4746 void CodeGenerator::recordSyncPoint(Asm& as, SyncOptions sync) {
4747   recordSyncPoint(Vauto().main(as), sync);
4748 }
4749
4750 void CodeGenerator::cgLdMIStateAddr(IRInstruction* inst) {
4751   auto base = srcLoc(0).reg();
4752   int64_t offset = inst->src(1)->intVal();
4753   m_as.lea (base[offset], dstLoc(0).reg());
4754 }
4755
4756 void CodeGenerator::cgLdLoc(IRInstruction* inst) {
4757   cgLoad(inst->dst(), dstLoc(0),
4758          srcLoc(0).reg()[localOffset(inst->extra<LdLoc>()->locId)]);
4759 }
4760
4761 void CodeGenerator::cgLdLocAddr(IRInstruction* inst) {
4762   auto const fpReg  = srcLoc(0).reg();
4763   auto const offset = localOffset(inst->extra<LdLocAddr>()->locId);
4764   if (dstLoc(0).hasReg()) {
4765     m_as.lea(fpReg[offset], dstLoc(0).reg());
4766   }
4767 }
4768
4769 void CodeGenerator::cgLdGbl(IRInstruction* inst) {
4770   cgLoad(
4771     inst->dst(),
4772     dstLoc(0),
4773     srcLoc(0).reg()[localOffset(inst->extra<LdGbl>()->locId)],
4774     inst->taken()
4775   );
4776 }
4777
4778 void CodeGenerator::cgStGbl(IRInstruction* inst) {
4779   auto ptr = srcLoc(0).reg();
4780   auto off = localOffset(inst->extra<StGbl>()->locId);
4781   cgStore(ptr[off], inst->src(1), srcLoc(1), Width::Full);
4782 }
4783
4784 void CodeGenerator::cgLdStackAddr(IRInstruction* inst) {
4785   auto const base   = srcLoc(0).reg();
4786   auto const offset = cellsToBytes(inst->extra<LdStackAddr>()->offset);
4787   auto const dst    = dstLoc(0).reg();
4788   m_as.lea (base[offset], dst);
4789 }
4790
4791 void CodeGenerator::cgLdStack(IRInstruction* inst) {
4792   assert(inst->taken() == nullptr);
4793   cgLoad(inst->dst(), dstLoc(0),
4794          srcLoc(0).reg()[cellsToBytes(inst->extra<LdStack>()->offset)]);
4795 }
4796
4797 void CodeGenerator::cgGuardStk(IRInstruction* inst) {
4798   auto const rSP = srcLoc(0).reg();
4799   auto const baseOff = cellsToBytes(inst->extra<GuardStk>()->offset);
4800   emitTypeGuard(inst->typeParam(),
4801                 rSP[baseOff + TVOFF(m_type)],
4802                 rSP[baseOff + TVOFF(m_data)]);
4803 }
4804
4805 void CodeGenerator::cgCheckStk(IRInstruction* inst) {
4806   auto const rbase = srcLoc(0).reg();
4807   auto const baseOff = cellsToBytes(inst->extra<CheckStk>()->offset);
4808
4809   emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
4810                 rbase[baseOff + TVOFF(m_data)], inst->taken());
4811 }
4812
4813 void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
4814   auto const rFP = srcLoc(0).reg();
4815   auto const baseOff = localOffset(inst->extra<GuardLoc>()->locId);
4816   emitTypeGuard(inst->typeParam(),
4817                 rFP[baseOff + TVOFF(m_type)],
4818                 rFP[baseOff + TVOFF(m_data)]);
4819 }
4820
4821 void CodeGenerator::cgCheckLoc(IRInstruction* inst) {
4822   auto const rbase = srcLoc(0).reg();
4823   auto const baseOff = localOffset(inst->extra<CheckLoc>()->locId);
4824   emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
4825                 rbase[baseOff + TVOFF(m_data)], inst->taken());
4826 }
4827
4828 template<class Loc>
4829 void CodeGenerator::emitSideExitGuard(Type type,
4830                                       Loc typeSrc,
4831                                       Loc dataSrc,
4832                                       Offset taken) {
4833   emitTypeTest(
4834     type, typeSrc, dataSrc,
4835     [&](ConditionCode cc) {
4836       auto const sk = SrcKey(curFunc(), taken, resumed());
4837       emitBindSideExit(m_mainCode, m_frozenCode, ccNegate(cc), sk);
4838     });
4839 }
4840
4841 void CodeGenerator::cgSideExitGuardLoc(IRInstruction* inst) {
4842   auto const fp    = srcLoc(0).reg();
4843   auto const extra = inst->extra<SideExitGuardLoc>();
4844   emitSideExitGuard(inst->typeParam(),
4845                     fp[localOffset(extra->checkedSlot) + TVOFF(m_type)],
4846                     fp[localOffset(extra->checkedSlot) + TVOFF(m_data)],
4847                     extra->taken);
4848 }
4849
4850 void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) {
4851   auto const sp    = srcLoc(0).reg();
4852   auto const extra = inst->extra<SideExitGuardStk>();
4853
4854   emitSideExitGuard(inst->typeParam(),
4855                     sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)],
4856                     sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_data)],
4857                     extra->taken);
4858 }
4859
4860 void CodeGenerator::cgExitJcc(IRInstruction* inst) {
4861   auto const extra = inst->extra<SideExitJccData>();
4862   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
4863   auto& v = vmain();
4864   emitCompare(v, inst);
4865   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
4866 }
4867
4868 void CodeGenerator::cgExitJccInt(IRInstruction* inst) {
4869   auto const extra = inst->extra<SideExitJccData>();
4870   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
4871   auto& v = vmain();
4872   emitCompareInt(v, inst);
4873   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
4874 }
4875
4876 void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) {
4877   assert(dstLoc(0).reg() == rsp);
4878 }
4879
4880 void CodeGenerator::cgCheckType(IRInstruction* inst) {
4881   auto const src   = inst->src(0);
4882   auto const rData = srcLoc(0).reg(0);
4883   auto const rType = srcLoc(0).reg(1);
4884   assert(rData != InvalidReg);
4885
4886   auto doJcc = [&](ConditionCode cc) {
4887     emitFwdJcc(ccNegate(cc), inst->taken());
4888   };
4889   auto doMov = [&]() {
4890     auto const valDst = dstLoc(0).reg(0);
4891     auto const typeDst = dstLoc(0).reg(1);
4892     // TODO: #3626251: XLS: Let Uses say whether a constant is
4893     // allowed, and if not, assign a register.
4894     if (valDst != InvalidReg) {
4895       emitMovRegReg(m_as, rData, valDst);
4896     }
4897     if (typeDst != InvalidReg) {
4898       if (rType != InvalidReg) emitMovRegReg(m_as, rType, typeDst);
4899       else m_as.emitImmReg(src->type().toDataType(), typeDst);
4900     }
4901   };
4902
4903   Type typeParam = inst->typeParam();
4904   // CheckTypes that are known to succeed or fail may be kept around
4905   // by the simplifier in case the guard can be relaxed.
4906   if (src->isA(typeParam)) {
4907     // src is the target type or better. do nothing.
4908     doMov();
4909     return;
4910   } else if (src->type().not(typeParam)) {
4911     // src is definitely not the target type. always jump.
4912     emitFwdJmp(m_as, inst->taken(), m_state);
4913     return;
4914   }
4915
4916   if (rType != InvalidReg) {
4917     emitTypeTest(typeParam, rType, rData, doJcc);
4918   } else {
4919     Type srcType = src->type();
4920     if (srcType.isBoxed() && typeParam.isBoxed()) {
4921       // Nothing to do here, since we check the inner type at the uses
4922     } else if (typeParam.isSpecialized()) {
4923       // We're just checking the array kind or object class of a value with a
4924       // mostly-known type.
4925       emitSpecializedTypeTest(typeParam, rData, doJcc);
4926     } else if (typeParam <= Type::Uncounted &&
4927                ((srcType <= Type::Str && typeParam.maybe(Type::StaticStr)) ||
4928                 (srcType <= Type::Arr && typeParam.maybe(Type::StaticArr)))) {
4929       // We carry Str and Arr operands around without a type register,
4930       // even though they're union types.  The static and non-static
4931       // subtypes are distinguised by the refcount field.
4932       m_as.cmpl(0, rData[FAST_REFCOUNT_OFFSET]);
4933       doJcc(CC_L);
4934     } else {
4935       // We should only get here if this CheckType should've been simplified
4936       // away but wasn't for some reason, so do a simple version of what it
4937       // would've. Widen inner types first since CheckType ignores them.
4938       if (srcType.maybeBoxed()) srcType |= Type::BoxedCell;
4939       if (typeParam.maybeBoxed()) typeParam |= Type::BoxedCell;
4940
4941       if (srcType <= typeParam) {
4942         // This will always succeed. Do nothing.
4943       } else if (srcType.not(typeParam)) {
4944         // This will always fail. Emit an unconditional jmp.
4945         emitFwdJmp(m_as, inst->taken(), m_state);
4946         return;
4947       } else {
4948         always_assert_log(
4949           false,
4950           [&] {
4951             return folly::format("Bad src: {} and dst: {} types in '{}'",
4952                                  srcType, typeParam, *inst).str();
4953           });
4954       }
4955     }
4956   }
4957
4958   doMov();
4959 }
4960
4961 void CodeGenerator::cgCheckTypeMem(IRInstruction* inst) {
4962   auto const reg = srcLoc(0).reg();
4963   emitTypeCheck(inst->typeParam(), reg[TVOFF(m_type)],
4964                 reg[TVOFF(m_data)], inst->taken());
4965 }
4966
4967 void CodeGenerator::cgCheckDefinedClsEq(IRInstruction* inst) {
4968   auto const clsName = inst->extra<CheckDefinedClsEq>()->clsName;
4969   auto const cls     = inst->extra<CheckDefinedClsEq>()->cls;
4970   auto const ch      = NamedEntity::get(clsName)->getClassHandle();
4971   auto clsImm = Immed64(cls);
4972   if (clsImm.fits(sz::dword)) {
4973     m_as.cmpq(clsImm.l(), rVmTl[ch]);
4974   } else {
4975     m_as.emitImmReg(cls, m_rScratch);
4976     m_as.cmpq(m_rScratch, rVmTl[ch]);
4977   }
4978   emitFwdJcc(CC_NZ, inst->taken());
4979 }
4980
4981 void CodeGenerator::cgGuardRefs(IRInstruction* inst) {
4982   assert(inst->numSrcs() == 5);
4983
4984   DEBUG_ONLY SSATmp* nParamsTmp = inst->src(1);
4985   SSATmp* firstBitNumTmp = inst->src(2);
4986   SSATmp* mask64Tmp  = inst->src(3);
4987   SSATmp* vals64Tmp  = inst->src(4);
4988
4989   auto funcPtrReg = srcLoc(0).reg();
4990   auto nParamsReg = srcLoc(1).reg();
4991   auto mask64Reg = srcLoc(3).reg();
4992   auto vals64Reg = srcLoc(4).reg();
4993
4994   // Get values in place
4995   assert(funcPtrReg != InvalidReg);
4996
4997   assert(nParamsReg != InvalidReg || nParamsTmp->isConst());
4998
4999   assert(firstBitNumTmp->isConst(Type::Int));
5000   auto firstBitNum = safe_cast<int32_t>(firstBitNumTmp->intVal());
5001
5002   uint64_t mask64 = mask64Tmp->intVal();
5003   assert(mask64Reg != InvalidReg || mask64 == uint32_t(mask64));
5004   assert(mask64);
5005
5006   uint64_t vals64 = vals64Tmp->intVal();
5007   assert(vals64Reg != InvalidReg || vals64 == uint32_t(vals64));
5008   assert((vals64 & mask64) == vals64);
5009
5010   auto const destSK = SrcKey(curFunc(), m_unit.bcOff(), resumed());
5011   auto const destSR = mcg->tx().getSrcRec(destSK);
5012
5013   auto thenBody = [&](Asm& a) {
5014     auto bitsOff = sizeof(uint64_t) * (firstBitNum / 64);
5015     auto cond = CC_NE;
5016     auto bitsPtrReg = m_rScratch;
5017
5018     if (firstBitNum == 0) {
5019       bitsOff = Func::refBitValOff();
5020       bitsPtrReg = funcPtrReg;
5021     } else {
5022       a.loadq(funcPtrReg[Func::sharedOff()], bitsPtrReg);
5023       bitsOff -= sizeof(uint64_t);
5024     }
5025
5026     if (vals64 == 0 || (mask64 & (mask64 - 1)) == 0) {
5027       // If vals64 is zero, or we're testing a single
5028       // bit, we can get away with a single test,
5029       // rather than mask-and-compare
5030       if (mask64Reg != InvalidReg) {
5031         a.testq  (mask64Reg, bitsPtrReg[bitsOff]);
5032       } else {
5033         if (mask64 < 256) {
5034           a.testb((int8_t)mask64, bitsPtrReg[bitsOff]);
5035         } else {
5036           a.testl((int32_t)mask64, bitsPtrReg[bitsOff]);
5037         }
5038       }
5039       if (vals64) cond = CC_E;
5040     } else {
5041       auto bitsValReg = m_rScratch;
5042       a.  loadq  (bitsPtrReg[bitsOff], bitsValReg);
5043       if (debug) bitsPtrReg = InvalidReg;
5044
5045       //     bitsValReg <- bitsValReg & mask64
5046       if (mask64Reg != InvalidReg) {
5047         a.  andq   (mask64Reg, bitsValReg);
5048       } else if (mask64 < 256) {
5049         a.  andb   ((int8_t)mask64, rbyte(bitsValReg));
5050       } else {
5051         a.  andl   ((int32_t)mask64, r32(bitsValReg));
5052       }
5053
5054       //   If bitsValReg != vals64, then goto Exit
5055       if (vals64Reg != InvalidReg) {
5056         a.  cmpq   (vals64Reg, bitsValReg);
5057       } else if (mask64 < 256) {
5058         assert(vals64 < 256);
5059         a.  cmpb   ((int8_t)vals64, rbyte(bitsValReg));
5060       } else {
5061         a.  cmpl   ((int32_t)vals64, r32(bitsValReg));
5062       }
5063     }
5064     destSR->emitFallbackJump(m_mainCode, cond);
5065   };
5066
5067   if (firstBitNum == 0) {
5068     assert(nParamsReg == InvalidReg);
5069     // This is the first 64 bits. No need to check
5070     // nParams.
5071     thenBody(m_as);
5072   } else {
5073     assert(nParamsReg != InvalidReg);
5074     // Check number of args...
5075     m_as.    cmpq   (firstBitNum, nParamsReg);
5076
5077     if (vals64 != 0 && vals64 != mask64) {
5078       // If we're beyond nParams, then either all params
5079       // are refs, or all params are non-refs, so if vals64
5080       // isn't 0 and isnt mask64, there's no possibility of
5081       // a match
5082       destSR->emitFallbackJump(m_mainCode, CC_LE);
5083       thenBody(m_as);
5084     } else {
5085       ifThenElse(CC_NLE, thenBody, /* else */ [&](Asm& a) {
5086         //   If not special builtin...
5087         a.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
5088         destSR->emitFallbackJump(m_mainCode, vals64 ? CC_Z : CC_NZ);
5089       });
5090     }
5091   }
5092 }
5093
5094 void CodeGenerator::cgLdPropAddr(IRInstruction* inst) {
5095   auto const dstReg = dstLoc(0).reg();
5096   auto const objReg = srcLoc(0).reg();
5097   auto const prop = inst->src(1);
5098   auto& a = m_as;
5099   always_assert(objReg != InvalidReg);
5100   always_assert(dstReg != InvalidReg);
5101   a.    lea    (objReg[prop->intVal()], dstReg);
5102 }
5103
5104 void CodeGenerator::cgLdClsMethod(IRInstruction* inst) {
5105   auto dstReg = dstLoc(0).reg();
5106   auto clsReg = srcLoc(0).reg();
5107   int32_t mSlotVal = inst->src(1)->rawVal();
5108
5109   assert(dstReg != InvalidReg);
5110
5111   auto methOff = int32_t(mSlotVal * sizeof(Func*));
5112   m_as.loadq(clsReg[methOff],  dstReg);
5113 }
5114
5115 void CodeGenerator::cgLookupClsMethodCache(IRInstruction* inst) {
5116   auto funcDestReg   = dstLoc(0).reg(0);
5117
5118   auto const& extra = *inst->extra<ClsMethodData>();
5119   auto const cls = extra.clsName;
5120   auto const method = extra.methodName;
5121   auto const ne = extra.namedEntity;
5122   auto const ch = StaticMethodCache::alloc(cls,
5123                                            method,
5124                                            getContextName(curClass()));
5125
5126   if (false) { // typecheck
5127     UNUSED TypedValue* fake_fp = nullptr;
5128     UNUSED TypedValue* fake_sp = nullptr;
5129     const UNUSED Func* f = StaticMethodCache::lookup(
5130       ch, ne, cls, method, fake_fp);
5131   }
5132   if (inst->src(0)->isConst()) {
5133     PUNT(LookupClsMethodCache_const_fp);
5134   }
5135
5136   // can raise an error if class is undefined
5137   cgCallHelper(m_as,
5138                CppCall::direct(StaticMethodCache::lookup),
5139                callDest(funcDestReg),
5140                SyncOptions::kSyncPoint,
5141                argGroup()
5142                  .imm(ch)       // Handle ch
5143                  .immPtr(ne)            // NamedEntity* np.second
5144                  .immPtr(cls)           // className
5145                  .immPtr(method)        // methodName
5146                  .reg(srcLoc(0).reg()) // frame pointer
5147               );
5148 }
5149
5150 void CodeGenerator::cgLdClsMethodCacheCommon(IRInstruction* inst, Offset off) {
5151   auto dstReg = dstLoc(0).reg();
5152   if (dstReg == InvalidReg) return;
5153
5154   auto const& extra = *inst->extra<ClsMethodData>();
5155   auto const clsName = extra.clsName;
5156   auto const methodName = extra.methodName;
5157   auto const ch = StaticMethodCache::alloc(clsName, methodName,
5158                                            getContextName(curClass()));
5159   if (dstReg != InvalidReg) {
5160     m_as.loadq(rVmTl[ch + off], dstReg);
5161   }
5162 }
5163
5164 void CodeGenerator::cgLdClsMethodCacheFunc(IRInstruction* inst) {
5165   cgLdClsMethodCacheCommon(inst, offsetof(StaticMethodCache, m_func));
5166
5167 }
5168
5169 void CodeGenerator::cgLdClsMethodCacheCls(IRInstruction* inst) {
5170   cgLdClsMethodCacheCommon(inst, offsetof(StaticMethodCache, m_cls));
5171 }
5172
5173 /**
5174  * Helper to emit getting the value for ActRec's m_this/m_cls slot
5175  * from a This pointer depending on whether the callee method is
5176  * static or not.
5177  */
5178 void CodeGenerator::emitGetCtxFwdCallWithThis(PhysReg ctxReg,
5179                                               bool    staticCallee) {
5180   if (staticCallee) {
5181     // Load (this->m_cls | 0x1) into ctxReg.
5182     emitLdLowPtr(m_as, ctxReg[ObjectData::getVMClassOffset()],
5183                  ctxReg, sizeof(LowClassPtr));
5184     m_as.orq(1, ctxReg);
5185   } else {
5186     // Just incref $this.
5187     emitIncRef(m_as, ctxReg);
5188   }
5189 }
5190
5191 /**
5192  * This method is similar to emitGetCtxFwdCallWithThis above, but
5193  * whether or not the callee is a static method is unknown at JIT
5194  * time, and that is determined dynamically by looking up into the
5195  * StaticMethodFCache.
5196  */
5197 void CodeGenerator::cgGetCtxFwdCall(IRInstruction* inst) {
5198   PhysReg destCtxReg = dstLoc(0).reg(0);
5199   auto srcCtxTmp = inst->src(0);
5200   auto srcCtxReg = srcLoc(0).reg(0);
5201   const Func* callee = inst->src(1)->funcVal();
5202   bool      withThis = srcCtxTmp->isA(Type::Obj);
5203
5204   // Eagerly move src into the dest reg
5205   emitMovRegReg(m_as, srcCtxReg, destCtxReg);
5206
5207   Label End;
5208   // If we don't know whether we have a This, we need to check dynamically
5209   if (!withThis) {
5210     m_as.testb(1, rbyte(destCtxReg));
5211     m_as.jcc8(CC_NZ, End);
5212   }
5213
5214   // If we have a This pointer in destCtxReg, then select either This
5215   // or its Class based on whether callee is static or not
5216   emitGetCtxFwdCallWithThis(destCtxReg, (callee->attrs() & AttrStatic));
5217
5218   asm_label(m_as, End);
5219 }
5220
5221 void CodeGenerator::cgLdClsMethodFCacheFunc(IRInstruction* inst) {
5222   auto const& extra     = *inst->extra<ClsMethodData>();
5223   auto const clsName    = extra.clsName;
5224   auto const methodName = extra.methodName;
5225   auto const dstReg     = dstLoc(0).reg();
5226
5227   auto const ch = StaticMethodFCache::alloc(
5228     clsName, methodName, getContextName(curClass())
5229   );
5230   if (dstReg != InvalidReg) {
5231     m_as.loadq(rVmTl[ch], dstReg);
5232   }
5233 }
5234
5235 void CodeGenerator::cgLookupClsMethodFCache(IRInstruction* inst) {
5236   auto const funcDestReg = dstLoc(0).reg(0);
5237   auto const cls         = inst->src(0)->clsVal();
5238   auto const& extra      = *inst->extra<ClsMethodData>();
5239   auto const methName    = extra.methodName;
5240   auto const fpReg       = srcLoc(1).reg();
5241   auto const clsName     = cls->name();
5242
5243   auto ch = StaticMethodFCache::alloc(
5244     clsName, methName, getContextName(curClass())
5245   );
5246
5247   const Func* (*lookup)(
5248     RDS::Handle, const Class*, const StringData*, TypedValue*) =
5249     StaticMethodFCache::lookup;
5250   cgCallHelper(m_as,
5251                CppCall::direct(lookup),
5252                callDest(funcDestReg),
5253                SyncOptions::kSyncPoint,
5254                argGroup()
5255                  .imm(ch)
5256                  .immPtr(cls)
5257                  .immPtr(methName)
5258                  .reg(fpReg));
5259 }
5260
5261 void CodeGenerator::emitGetCtxFwdCallWithThisDyn(PhysReg      destCtxReg,
5262                                                  PhysReg      thisReg,
5263                                                  RDS::Handle ch) {
5264   Label NonStaticCall, End;
5265
5266   // thisReg is holding $this. Should we pass it to the callee?
5267   m_as.cmpl(1,
5268             rVmTl[ch + offsetof(StaticMethodFCache, m_static)]);
5269   m_as.jcc8(CC_NE, NonStaticCall);
5270   // If calling a static method...
5271   {
5272     // Load (this->m_cls | 0x1) into destCtxReg
5273     emitLdLowPtr(m_as, thisReg[ObjectData::getVMClassOffset()],
5274                  destCtxReg, sizeof(LowClassPtr));
5275     m_as.orq(1, destCtxReg);
5276     m_as.jmp8(End);
5277   }
5278   // Else: calling non-static method
5279   {
5280     asm_label(m_as, NonStaticCall);
5281     emitMovRegReg(m_as, thisReg, destCtxReg);
5282     emitIncRef(m_as, destCtxReg);
5283   }
5284   asm_label(m_as, End);
5285 }
5286
5287 void CodeGenerator::cgGetCtxFwdCallDyn(IRInstruction* inst) {
5288   auto srcCtxTmp  = inst->src(0);
5289   auto srcCtxReg  = srcLoc(0).reg();
5290   auto destCtxReg = dstLoc(0).reg();
5291
5292   Label End;
5293
5294   emitMovRegReg(m_as, srcCtxReg, destCtxReg);
5295   auto const t = srcCtxTmp->type();
5296   if (t <= Type::Cctx) {
5297     // Nothing to do. Forward the context as is.
5298     return;
5299   } else if (t <= Type::Obj) {
5300     // We definitely have $this, so always run code emitted by
5301     // emitGetCtxFwdCallWithThisDyn
5302   } else {
5303     assert(t <= Type::Ctx);
5304     // dynamically check if we have a This pointer and call
5305     // emitGetCtxFwdCallWithThisDyn below
5306     m_as.testb(1, rbyte(destCtxReg));
5307     m_as.jcc8(CC_NZ, End);
5308   }
5309
5310   // If we have a 'this' pointer ...
5311   auto const& extra = *inst->extra<ClsMethodData>();
5312   auto const ch = StaticMethodFCache::alloc(
5313     extra.clsName, extra.methodName, getContextName(curClass())
5314   );
5315   emitGetCtxFwdCallWithThisDyn(destCtxReg, destCtxReg, ch);
5316
5317   asm_label(m_as, End);
5318 }
5319
5320 void CodeGenerator::cgLdClsPropAddrKnown(IRInstruction* inst) {
5321   auto dstReg = dstLoc(0).reg();
5322
5323   auto cls  = inst->src(0)->clsVal();
5324   auto name = inst->src(1)->strVal();
5325
5326   auto ch = cls->sPropHandle(cls->lookupSProp(name));
5327   m_as.lea(rVmTl[ch], dstReg);
5328 }
5329
5330 RDS::Handle CodeGenerator::cgLdClsCachedCommon(
5331   IRInstruction* inst) {
5332   const StringData* className = inst->src(0)->strVal();
5333   auto ch = NamedEntity::get(className)->getClassHandle();
5334   auto dstReg = dstLoc(0).reg();
5335   if (dstReg == InvalidReg) {
5336     m_as. cmpq   (0, rVmTl[ch]);
5337   } else {
5338     m_as.  loadq  (rVmTl[ch], dstReg);
5339     m_as.  testq  (dstReg, dstReg);
5340   }
5341
5342   return ch;
5343 }
5344
5345 void CodeGenerator::cgLdClsCached(IRInstruction* inst) {
5346   auto ch = cgLdClsCachedCommon(inst);
5347   unlikelyIfBlock(CC_E, [&] (Asm& a) {
5348     Class* (*const func)(Class**, const StringData*) =
5349       JIT::lookupKnownClass;
5350     cgCallHelper(a,
5351                  CppCall::direct(func),
5352                  callDest(inst),
5353                  SyncOptions::kSyncPoint,
5354                  argGroup().addr(rVmTl, safe_cast<int32_t>(ch))
5355                            .ssa(0));
5356   });
5357 }
5358
5359 void CodeGenerator::cgLdClsCachedSafe(IRInstruction* inst) {
5360   cgLdClsCachedCommon(inst);
5361   if (Block* taken = inst->taken()) {
5362     emitFwdJcc(CC_Z, taken);
5363   }
5364 }
5365
5366 void CodeGenerator::cgDerefClsRDSHandle(IRInstruction* inst) {
5367   auto const dreg = dstLoc(0).reg();
5368   auto const ch   = inst->src(0);
5369   auto& a = m_as;
5370
5371   if (dreg == InvalidReg) return;
5372   if (ch->isConst()) {
5373     a.    loadq  (rVmTl[ch->rdsHandleVal()], dreg);
5374   } else {
5375     a.    loadq  (rVmTl[srcLoc(0).reg()], dreg);
5376   }
5377 }
5378
5379 void CodeGenerator::cgLdCls(IRInstruction* inst) {
5380   auto const ch = ClassCache::alloc();
5381   RDS::recordRds(ch, sizeof(ClassCache),
5382                  "ClassCache", curFunc()->fullName()->data());
5383
5384   cgCallHelper(m_as,
5385                CppCall::direct(ClassCache::lookup),
5386                callDest(inst),
5387                SyncOptions::kSyncPoint,
5388                argGroup().imm(ch).ssa(0/*className*/));
5389 }
5390
5391 void CodeGenerator::cgLdClsCns(IRInstruction* inst) {
5392   auto const extra = inst->extra<LdClsCns>();
5393   auto const link  = RDS::bindClassConstant(extra->clsName, extra->cnsName);
5394   cgLoad(inst->dst(), dstLoc(0), rVmTl[link.handle()], inst->taken());
5395 }
5396
5397 void CodeGenerator::cgLookupClsCns(IRInstruction* inst) {
5398   auto const extra = inst->extra<LookupClsCns>();
5399   auto const link  = RDS::bindClassConstant(extra->clsName, extra->cnsName);
5400   cgCallHelper(
5401     m_as,
5402     CppCall::direct(JIT::lookupClassConstantTv),
5403     callDestTV(inst),
5404     SyncOptions::kSyncPoint,
5405     argGroup()
5406       .addr(rVmTl, safe_cast<int32_t>(link.handle()))
5407       .immPtr(NamedEntity::get(extra->clsName))
5408       .immPtr(extra->clsName)
5409       .immPtr(extra->cnsName)
5410   );
5411 }
5412
5413 void CodeGenerator::cgLdCns(IRInstruction* inst) {
5414   const StringData* cnsName = inst->src(0)->strVal();
5415
5416   auto const ch = makeCnsHandle(cnsName, false);
5417   // Has an unlikely branch to a LookupCns
5418   cgLoad(inst->dst(), dstLoc(0), rVmTl[ch], inst->taken());
5419 }
5420
5421 void CodeGenerator::cgLookupCnsCommon(IRInstruction* inst) {
5422   SSATmp* cnsNameTmp = inst->src(0);
5423
5424   assert(cnsNameTmp->isConst(Type::StaticStr));
5425
5426   auto const cnsName = cnsNameTmp->strVal();
5427   auto const ch = makeCnsHandle(cnsName, false);
5428
5429   auto args = argGroup();
5430   args.addr(rVmTl, safe_cast<int32_t>(ch))
5431       .immPtr(cnsName)
5432       .imm(inst->op() == LookupCnsE);
5433
5434   cgCallHelper(m_as,
5435                CppCall::direct(lookupCnsHelper),
5436                callDestTV(inst),
5437                SyncOptions::kSyncPoint,
5438                args);
5439 }
5440
5441 void CodeGenerator::cgLookupCns(IRInstruction* inst) {
5442   cgLookupCnsCommon(inst);
5443 }
5444
5445 void CodeGenerator::cgLookupCnsE(IRInstruction* inst) {
5446   cgLookupCnsCommon(inst);
5447 }
5448
5449 void CodeGenerator::cgLookupCnsU(IRInstruction* inst) {
5450   SSATmp* cnsNameTmp = inst->src(0);
5451   SSATmp* fallbackNameTmp = inst->src(1);
5452
5453   const StringData* cnsName = cnsNameTmp->strVal();
5454
5455   const StringData* fallbackName = fallbackNameTmp->strVal();
5456   auto const fallbackCh = makeCnsHandle(fallbackName, false);
5457
5458   auto args = argGroup();
5459   args.addr(rVmTl, safe_cast<int32_t>(fallbackCh))
5460       .immPtr(cnsName)
5461       .immPtr(fallbackName);
5462
5463   cgCallHelper(m_as,
5464                CppCall::direct(lookupCnsUHelper),
5465                callDestTV(inst),
5466                SyncOptions::kSyncPoint,
5467                args);
5468 }
5469
5470 void CodeGenerator::cgAKExists(IRInstruction* inst) {
5471   SSATmp* arr = inst->src(0);
5472   SSATmp* key = inst->src(1);
5473
5474   bool (*obj_int_helper)(ObjectData*, int64_t) = &ak_exist_int_obj;
5475   bool (*obj_str_helper)(ObjectData*, StringData*) = &ak_exist_string_obj;
5476   bool (*arr_str_helper)(ArrayData*, StringData*) = &ak_exist_string;
5477
5478   if (key->type() <= Type::Null) {
5479     if (arr->isA(Type::Arr)) {
5480       cgCallHelper(m_as,
5481                    CppCall::direct(arr_str_helper),
5482                    callDest(inst),
5483                    SyncOptions::kNoSyncPoint,
5484                    argGroup().ssa(0/*arr*/).immPtr(staticEmptyString()));
5485     } else {
5486       m_as.movq(0, dstLoc(0).reg());
5487     }
5488     return;
5489   }
5490
5491   auto helper_func = arr->isA(Type::Obj)
5492     ? (key->isA(Type::Int)
5493        ? CppCall::direct(obj_int_helper)
5494        : CppCall::direct(obj_str_helper))
5495     : (key->isA(Type::Int)
5496        ? CppCall::array(&g_array_funcs.existsInt)
5497        : CppCall::direct(arr_str_helper));
5498
5499   cgCallHelper(m_as,
5500                helper_func,
5501                callDest(inst),
5502                SyncOptions::kNoSyncPoint,
5503                argGroup().ssa(0/*arr*/).ssa(1/*key*/));
5504 }
5505
5506 void CodeGenerator::cgLdGblAddr(IRInstruction* inst) {
5507   auto dstReg = dstLoc(0).reg();
5508   cgCallHelper(m_as,
5509                CppCall::direct(ldGblAddrHelper),
5510                callDest(dstReg),
5511                SyncOptions::kNoSyncPoint,
5512                argGroup().ssa(0));
5513   m_as.testq(dstReg, dstReg);
5514   emitFwdJcc(CC_Z, inst->taken());
5515 }
5516
5517 void CodeGenerator::emitTestZero(SSATmp* src, PhysLoc srcLoc) {
5518   emitTestZero(Vauto().main(m_as), src, srcLoc);
5519 }
5520
5521 void CodeGenerator::emitTestZero(Vout& v, SSATmp* src, PhysLoc srcLoc) {
5522   auto reg = srcLoc.reg();
5523
5524   /*
5525    * If src is const, normally a earlier optimization pass should have
5526    * converted the thing testing this condition into something
5527    * unconditional.  So rather than supporting constants efficiently
5528    * here, we just materialize the value into a register.
5529    */
5530   if (reg == InvalidReg) {
5531     reg = m_rScratch;
5532     v << ldimm{src->rawVal(), reg};
5533   }
5534
5535   if (src->isA(Type::Bool)) {
5536     v << testb{reg, reg};
5537   } else {
5538     v << testq{reg, reg};
5539   }
5540 }
5541
5542 void CodeGenerator::cgJmpZero(IRInstruction* inst) {
5543   emitTestZero(inst->src(0), srcLoc(0));
5544   emitFwdJcc(CC_Z, inst->taken());
5545 }
5546
5547 void CodeGenerator::cgJmpNZero(IRInstruction* inst) {
5548   emitTestZero(inst->src(0), srcLoc(0));
5549   emitFwdJcc(CC_NZ, inst->taken());
5550 }
5551
5552 void CodeGenerator::cgReqBindJmpZero(IRInstruction* inst) {
5553   // TODO(#2404427): prepareForTestAndSmash?
5554   auto& v = vmain();
5555   emitTestZero(v, inst->src(0), srcLoc(0));
5556   emitReqBindJcc(v, CC_Z, inst->extra<ReqBindJmpZero>());
5557 }
5558
5559 void CodeGenerator::cgReqBindJmpNZero(IRInstruction* inst) {
5560   // TODO(#2404427): prepareForTestAndSmash?
5561   auto& v = vmain();
5562   emitTestZero(v, inst->src(0), srcLoc(0));
5563   emitReqBindJcc(v, CC_NZ, inst->extra<ReqBindJmpNZero>());
5564 }
5565
5566 void CodeGenerator::cgSideExitJmpZero(IRInstruction* inst) {
5567   auto const extra = inst->extra<SideExitJccData>();
5568   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
5569   auto& v = vmain();
5570   emitTestZero(v, inst->src(0), srcLoc(0));
5571   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
5572 }
5573
5574 void CodeGenerator::cgSideExitJmpNZero(IRInstruction* inst) {
5575   auto const extra = inst->extra<SideExitJccData>();
5576   auto const sk = SrcKey(curFunc(), extra->taken, resumed());
5577   auto& v = vmain();
5578   emitTestZero(v, inst->src(0), srcLoc(0));
5579   v << bindexit{opToConditionCode(inst->op()), sk, extra->trflags};
5580 }
5581
5582 void CodeGenerator::cgJmp(IRInstruction* inst) {
5583   if (!m_state.noTerminalJmp) {
5584     emitFwdJmp(m_as, inst->taken(), m_state);
5585   }
5586 }
5587
5588 void CodeGenerator::cgJmpIndirect(IRInstruction* inst) {
5589   auto& v = vmain();
5590   v << jmpr{srcLoc(0).reg()};
5591 }
5592
5593 void CodeGenerator::cgCheckInit(IRInstruction* inst) {
5594   Block* label = inst->taken();
5595   assert(label);
5596   SSATmp* src = inst->src(0);
5597
5598   if (src->type().not(Type::Uninit)) return;
5599
5600   auto typeReg = srcLoc(0).reg(1);
5601   assert(typeReg != InvalidReg);
5602
5603   static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0");
5604   m_as.testb (rbyte(typeReg), rbyte(typeReg));
5605   emitFwdJcc(CC_Z, label);
5606 }
5607
5608 void CodeGenerator::cgCheckInitMem(IRInstruction* inst) {
5609   Block* label = inst->taken();
5610   assert(label);
5611   SSATmp* base = inst->src(0);
5612   int64_t offset = inst->src(1)->intVal();
5613   Type t = base->type().deref();
5614   if (t.not(Type::Uninit)) return;
5615   auto basereg = srcLoc(0).reg();
5616   emitCmpTVType(m_as, KindOfUninit, basereg[offset + TVOFF(m_type)]);
5617   emitFwdJcc(CC_Z, label);
5618 }
5619
5620 void CodeGenerator::cgCheckSurpriseFlags(IRInstruction* inst) {
5621   emitTestSurpriseFlags(m_as);
5622   emitFwdJcc(CC_NZ, inst->taken());
5623 }
5624
5625 void CodeGenerator::cgCheckCold(IRInstruction* inst) {
5626   Block*     label = inst->taken();
5627   TransID  transId = inst->extra<CheckCold>()->transId;
5628   auto counterAddr = mcg->tx().profData()->transCounterAddr(transId);
5629
5630   emitLoadImm(m_as, uint64_t(counterAddr), m_rScratch);
5631   m_as.decq(m_rScratch[0]);
5632   emitFwdJcc(CC_LE, label);
5633 }
5634
5635 static const StringData* s_ReleaseVV = makeStaticString("ReleaseVV");
5636
5637 void CodeGenerator::cgReleaseVVOrExit(IRInstruction* inst) {
5638   auto* const label = inst->taken();
5639   auto const rFp = srcLoc(0).reg();
5640
5641   TargetProfile<ReleaseVVProfile> profile(m_unit.context(), m_curInst->marker(),
5642                                           s_ReleaseVV);
5643   if (profile.profiling()) {
5644     m_as.incw(rVmTl[profile.handle() + offsetof(ReleaseVVProfile, executed)]);
5645   }
5646
5647   m_as.cmpq(0, rFp[AROFF(m_varEnv)]);
5648
5649   bool releaseUnlikely = true;
5650   if (profile.optimizing()) {
5651     auto const data = profile.data(ReleaseVVProfile::reduce);
5652     FTRACE(3, "cgReleaseVVOrExit({}): percentReleased = {}\n",
5653            inst->toString(), data.percentReleased());
5654     if (data.percentReleased() >= RuntimeOption::EvalJitPGOReleaseVVMinPercent)
5655     {
5656       releaseUnlikely = false;
5657     }
5658   }
5659   ifBlock(CC_NZ, [&] (Asm& a) {
5660       if (profile.profiling()) {
5661         a.incw(rVmTl[profile.handle() + offsetof(ReleaseVVProfile, released)]);
5662       }
5663       a.testl(ActRec::kExtraArgsBit, rFp[AROFF(m_varEnv)]);
5664       emitFwdJcc(a, CC_Z, label);
5665       cgCallHelper(
5666         a,
5667         CppCall::direct(static_cast<void (*)(ActRec*)>(ExtraArgs::deallocate)),
5668         kVoidDest,
5669         SyncOptions::kSyncPoint,
5670         argGroup().reg(rFp)
5671       );
5672     },
5673     releaseUnlikely);
5674 }
5675
5676 void CodeGenerator::cgBoxPtr(IRInstruction* inst) {
5677   auto base    = srcLoc(0).reg();
5678   auto dstReg  = dstLoc(0).reg();
5679   emitMovRegReg(m_as, base, dstReg);
5680   emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)],
5681                base[TVOFF(m_data)],
5682     [&](ConditionCode cc) {
5683       ifThen(m_as, ccNegate(cc), [&](Asm& a) {
5684         cgCallHelper(a,
5685                      CppCall::direct(tvBox),
5686                      callDest(dstReg),
5687                      SyncOptions::kNoSyncPoint,
5688                      argGroup().ssa(0/*addr*/));
5689       });
5690     });
5691 }
5692
5693 void CodeGenerator::cgConcatCellCell(IRInstruction* inst) {
5694   // Supported cases are all simplified into other instructions
5695   CG_PUNT(cgConcatCellCell);
5696 }
5697
5698 void CodeGenerator::cgInterpOneCommon(IRInstruction* inst) {
5699   int64_t pcOff = inst->extra<InterpOneData>()->bcOff;
5700
5701   auto opc = *(curFunc()->unit()->at(pcOff));
5702   void* interpOneHelper = interpOneEntryPoints[opc];
5703
5704   auto dstReg = InvalidReg;
5705   if (inst->src(1)->isConst()) {
5706     PUNT(InterpOneCommon_const_fp);
5707   }
5708   cgCallHelper(m_as,
5709                CppCall::direct(reinterpret_cast<void (*)()>(interpOneHelper)),
5710                callDest(dstReg),
5711                SyncOptions::kSyncPoint,
5712                argGroup().ssa(1/*fp*/).ssa(0/*sp*/).imm(pcOff));
5713 }
5714
5715 void CodeGenerator::cgInterpOne(IRInstruction* inst) {
5716   cgInterpOneCommon(inst);
5717
5718   auto const& extra = *inst->extra<InterpOne>();
5719   auto newSpReg = dstLoc(0).reg();
5720   assert(newSpReg == srcLoc(0).reg());
5721
5722   auto spAdjustBytes = cellsToBytes(extra.cellsPopped - extra.cellsPushed);
5723   if (spAdjustBytes != 0) {
5724     m_as.addq(spAdjustBytes, newSpReg);
5725   }
5726 }
5727
5728 void CodeGenerator::cgInterpOneCF(IRInstruction* inst) {
5729   cgInterpOneCommon(inst);
5730
5731   m_as.loadq(rVmTl[RDS::kVmfpOff], rVmFp);
5732   m_as.loadq(rVmTl[RDS::kVmspOff], rVmSp);
5733
5734   emitServiceReq(m_mainCode, REQ_RESUME);
5735 }
5736
5737 void CodeGenerator::cgContEnter(IRInstruction* inst) {
5738   // ContEnter does not directly use SP, but the generator body we are jumping
5739   // to obviously does. We depend on SP via srcLoc(0) to avoid last SpillStack
5740   // be optimized away.
5741   auto curFpReg  = srcLoc(1).reg();
5742   auto genFpReg  = srcLoc(2).reg();
5743   auto addrReg   = srcLoc(3).reg();
5744   auto returnOff = safe_cast<int32_t>(inst->src(4)->intVal());
5745   assert(srcLoc(0).reg() == rVmSp);
5746   assert(curFpReg == rVmFp);
5747
5748   Label Prologue, End;
5749
5750   m_as.  storeq (curFpReg, genFpReg[AROFF(m_sfp)]);
5751   m_as.  storel (returnOff, genFpReg[AROFF(m_soff)]);
5752   m_as.  movq   (genFpReg, curFpReg);
5753   m_as.  jmp8   (End);
5754
5755   asm_label(m_as, Prologue);
5756   m_as.  pop    (curFpReg[AROFF(m_savedRip)]);
5757   m_as.  jmp    (addrReg);
5758
5759   asm_label(m_as, End);
5760   m_as.  call   (Prologue);
5761 }
5762
5763 void CodeGenerator::cgContPreNext(IRInstruction* inst) {
5764   auto contReg      = srcLoc(0).reg();
5765   auto checkStarted = inst->src(1)->boolVal();
5766   auto stateOff     = BaseGenerator::stateOff();
5767
5768   static_assert(uint8_t(BaseGenerator::State::Created) == 0, "used below");
5769   static_assert(uint8_t(BaseGenerator::State::Started) == 1, "used below");
5770
5771   // Take exit if state != 1 (checkStarted) or state > 1 (!checkStarted).
5772   m_as.cmpb(1, contReg[stateOff]);
5773   emitFwdJcc(checkStarted ? CC_NE : CC_A, inst->taken());
5774
5775   // Set generator state as Running.
5776   m_as.storeb(int8_t(BaseGenerator::State::Running), contReg[stateOff]);
5777 }
5778
5779 void CodeGenerator::cgContStartedCheck(IRInstruction* inst) {
5780   auto contReg  = srcLoc(0).reg();
5781   auto stateOff = BaseGenerator::stateOff();
5782
5783   static_assert(uint8_t(BaseGenerator::State::Created) == 0, "used below");
5784
5785   // Take exit if state == 0.
5786   m_as.testb(int8_t(0xff), contReg[stateOff]);
5787   emitFwdJcc(CC_Z, inst->taken());
5788 }
5789
5790 void CodeGenerator::cgContValid(IRInstruction* inst) {
5791   auto contReg  = srcLoc(0).reg();
5792   auto dstReg   = dstLoc(0).reg();
5793   auto stateOff = BaseGenerator::stateOff();
5794
5795   // Return 1 if generator state is not Done.
5796   m_as.cmpb(int8_t(BaseGenerator::State::Done), contReg[stateOff]);
5797   m_as.setne(rbyte(dstReg));
5798   m_as.movzbl(rbyte(dstReg), r32(dstReg));
5799 }
5800
5801 void CodeGenerator::cgContArIncKey(IRInstruction* inst) {
5802   auto contArReg = srcLoc(0).reg();
5803   m_as.incq(contArReg[CONTOFF(m_key) + TVOFF(m_data) -
5804                       c_Generator::arOff()]);
5805 }
5806
5807 void CodeGenerator::cgContArUpdateIdx(IRInstruction* inst) {
5808   auto contArReg = srcLoc(0).reg();
5809   int64_t off = CONTOFF(m_index) - c_Generator::arOff();
5810   auto newIdx = inst->src(1);
5811   auto newIdxReg = srcLoc(1).reg();
5812
5813   // this is hacky and awful oh god
5814   if (newIdx->isConst()) {
5815     auto const val = newIdx->rawVal();
5816     m_as.emitImmReg(val, m_rScratch);
5817     m_as.cmpq  (m_rScratch, contArReg[off]);
5818     m_as.cload_reg64_disp_reg64(CC_G, contArReg, off, m_rScratch);
5819   } else {
5820     m_as.loadq (contArReg[off], m_rScratch);
5821     m_as.cmpq  (m_rScratch, newIdxReg);
5822     m_as.cmov_reg64_reg64(CC_G, newIdxReg, m_rScratch);
5823   }
5824   m_as.storeq  (m_rScratch, contArReg[off]);
5825 }
5826
5827 void CodeGenerator::cgLdContActRec(IRInstruction* inst) {
5828   auto dest = dstLoc(0).reg();
5829   auto base = srcLoc(0).reg();
5830   ptrdiff_t offset = BaseGenerator::arOff();
5831
5832   m_as.lea (base[offset], dest) ;
5833 }
5834
5835 void CodeGenerator::emitLdRaw(IRInstruction* inst, size_t extraOff) {
5836   auto destReg = dstLoc(0).reg();
5837   auto offset  = inst->extra<RawMemData>()->info().offset;
5838   auto src     = srcLoc(0).reg()[offset + extraOff];
5839
5840   switch (inst->extra<RawMemData>()->info().size) {
5841     case sz::byte:  m_as.loadzbl(src, r32(destReg)); break;
5842     case sz::dword:
5843       m_as.loadl(src, r32(destReg));
5844       if (inst->extra<RawMemData>()->type == RawMemData::FuncNumParams) {
5845         // See Func::finishedEmittingParams and Func::numParams for rationale
5846         m_as.shrl(0x1, r32(destReg));
5847       }
5848       break;
5849     case sz::qword: m_as.loadq(src, destReg); break;
5850     default:        not_implemented();
5851   }
5852 }
5853
5854 void CodeGenerator::cgLdRaw(IRInstruction* inst) {
5855   emitLdRaw(inst, 0);
5856 }
5857
5858 void CodeGenerator::cgLdContArRaw(IRInstruction* inst) {
5859   emitLdRaw(inst, -BaseGenerator::arOff());
5860 }
5861
5862 void CodeGenerator::emitStRaw(IRInstruction* inst, size_t offset, int size) {
5863   auto dst    = srcLoc(0).reg()[offset];
5864   auto src    = inst->src(1);
5865   auto srcReg = srcLoc(1).reg();
5866
5867   if (srcReg == InvalidReg) {
5868     auto val = Immed64(src->type().hasRawVal() ? src->rawVal() : 0);
5869     switch (size) {
5870       case sz::byte:  m_as.storeb(val.b(), dst); break;
5871       case sz::dword: m_as.storel(val.l(), dst); break;
5872       case sz::qword: emitImmStoreq(m_as, val.q(), dst); break;
5873       default:        not_implemented();
5874     }
5875   } else {
5876     switch (size) {
5877       case sz::byte:  m_as.storeb(rbyte(srcReg), dst); break;
5878       case sz::dword: m_as.storel(r32(srcReg), dst); break;
5879       case sz::qword: m_as.storeq(srcReg, dst); break;
5880       default:        not_implemented();
5881     }
5882   }
5883 }
5884
5885 void CodeGenerator::cgStRaw(IRInstruction* inst) {
5886   auto const info = inst->extra<RawMemData>()->info();
5887   emitStRaw(inst, info.offset, info.size);
5888 }
5889
5890 void CodeGenerator::cgStContArRaw(IRInstruction* inst) {
5891   auto const info = inst->extra<RawMemData>()->info();
5892   emitStRaw(inst, -BaseGenerator::arOff() + info.offset, info.size);
5893 }
5894
5895 void CodeGenerator::cgLdContArValue(IRInstruction* inst) {
5896   auto contArReg = srcLoc(0).reg();
5897   const int64_t valueOff = CONTOFF(m_value);
5898   int64_t off = valueOff - c_Generator::arOff();
5899   cgLoad(inst->dst(), dstLoc(0), contArReg[off], inst->taken());
5900 }
5901
5902 void CodeGenerator::cgStContArValue(IRInstruction* inst) {
5903   auto contArReg = srcLoc(0).reg();
5904   auto value = inst->src(1);
5905   auto valueLoc = srcLoc(1);
5906   const int64_t valueOff = CONTOFF(m_value);
5907   int64_t off = valueOff - c_Generator::arOff();
5908   cgStore(contArReg[off], value, valueLoc, Width::Full);
5909 }
5910
5911 void CodeGenerator::cgLdContArKey(IRInstruction* inst) {
5912   auto contArReg = srcLoc(0).reg();
5913   const int64_t keyOff = CONTOFF(m_key);
5914   int64_t off = keyOff - c_Generator::arOff();
5915   cgLoad(inst->dst(), dstLoc(0), contArReg[off], inst->taken());
5916 }
5917
5918 void CodeGenerator::cgStContArKey(IRInstruction* inst) {
5919   auto contArReg = srcLoc(0).reg();
5920   auto value = inst->src(1);
5921   auto valueLoc = srcLoc(1);
5922
5923   const int64_t keyOff = CONTOFF(m_key);
5924   int64_t off = keyOff - c_Generator::arOff();
5925   cgStore(contArReg[off], value, valueLoc, Width::Full);
5926 }
5927
5928 void CodeGenerator::cgStAsyncArRaw(IRInstruction* inst) {
5929   auto const info = inst->extra<RawMemData>()->info();
5930   emitStRaw(inst, -c_AsyncFunctionWaitHandle::arOff() + info.offset,
5931             info.size);
5932 }
5933
5934 void CodeGenerator::cgStAsyncArResult(IRInstruction* inst) {
5935   auto asyncArReg = srcLoc(0).reg();
5936   auto value = inst->src(1);
5937   auto valueLoc = srcLoc(1);
5938   const int64_t off = c_AsyncFunctionWaitHandle::resultOff()
5939                     - c_AsyncFunctionWaitHandle::arOff();
5940   cgStore(asyncArReg[off], value, valueLoc, Width::Full);
5941 }
5942
5943 void CodeGenerator::cgLdAsyncArParentChain(IRInstruction* inst) {
5944   auto asyncArReg = srcLoc(0).reg();
5945   auto dstReg = dstLoc(0).reg();
5946   const int64_t off = c_AsyncFunctionWaitHandle::parentChainOff()
5947                     - c_AsyncFunctionWaitHandle::arOff();
5948   m_as.loadq(asyncArReg[off], dstReg);
5949 }
5950
5951 void CodeGenerator::cgAFWHBlockOn(IRInstruction* inst) {
5952   auto parentArReg = srcLoc(0).reg();
5953   auto childReg = srcLoc(1).reg();
5954   const int8_t blocked = c_WaitHandle::toKindState(
5955       c_WaitHandle::Kind::AsyncFunction, c_BlockableWaitHandle::STATE_BLOCKED);
5956   const int64_t firstParentOff = c_WaitableWaitHandle::parentChainOff()
5957                                + AsioBlockableChain::firstParentOff();
5958   const int64_t stateToArOff = c_AsyncFunctionWaitHandle::stateOff()
5959                              - c_AsyncFunctionWaitHandle::arOff();
5960   const int64_t nextParentToArOff = c_AsyncFunctionWaitHandle::blockableOff()
5961                                   + AsioBlockable::bitsOff()
5962                                   - c_AsyncFunctionWaitHandle::arOff();
5963   const int64_t childToArOff = c_AsyncFunctionWaitHandle::childOff()
5964                              - c_AsyncFunctionWaitHandle::arOff();
5965   const int64_t blockableToArOff = c_AsyncFunctionWaitHandle::blockableOff()
5966                                  - c_AsyncFunctionWaitHandle::arOff();
5967
5968   // parent->setState(STATE_BLOCKED);
5969   m_as.storeb(blocked, parentArReg[stateToArOff]);
5970
5971   // parent->m_blockable.m_bits = child->m_parentChain.m_firstParent|Kind::BWH;
5972   assert(uint8_t(AsioBlockable::Kind::BlockableWaitHandle) == 0);
5973   m_as.loadq (childReg[firstParentOff], m_rScratch);
5974   m_as.storeq(m_rScratch, parentArReg[nextParentToArOff]);
5975
5976   // child->m_parentChain.m_firstParent = &parent->m_blockable;
5977   m_as.lea   (parentArReg[blockableToArOff], m_rScratch);
5978   m_as.storeq(m_rScratch, childReg[firstParentOff]);
5979
5980   // parent->m_child = child;
5981   m_as.storeq(childReg, parentArReg[childToArOff]);
5982 }
5983
5984 void CodeGenerator::cgIsWaitHandle(IRInstruction* inst) {
5985   auto const robj = srcLoc(0).reg();
5986   auto const rdst = dstLoc(0).reg();
5987   auto& a = m_as;
5988
5989   static_assert(
5990     ObjectData::IsWaitHandle < 0xff,
5991     "we use byte instructions for IsWaitHandle"
5992   );
5993   a.   testb   (ObjectData::IsWaitHandle, robj[ObjectData::attributeOff()]);
5994   a.   setnz   (rbyte(rdst));
5995 }
5996
5997 void CodeGenerator::cgLdWHState(IRInstruction* inst) {
5998   auto const robj = srcLoc(0).reg();
5999   auto const rdst = dstLoc(0).reg();
6000   auto& a = m_as;
6001   a.    loadzbl (robj[ObjectData::whStateOffset()], r32(rdst));
6002   a.    andb    ((int8_t)0x0F, rbyte(rdst));
6003 }
6004
6005 void CodeGenerator::cgLdWHResult(IRInstruction* inst) {
6006   auto const robj = srcLoc(0).reg();
6007   cgLoad(inst->dst(), dstLoc(0), robj[c_WaitHandle::resultOff()]);
6008 }
6009
6010 void CodeGenerator::cgLdAFWHActRec(IRInstruction* inst) {
6011   auto const dest = dstLoc(0).reg();
6012   auto const base = srcLoc(0).reg();
6013   auto asyncArOffset = c_AsyncFunctionWaitHandle::arOff();
6014   m_as.lea (base[asyncArOffset], dest);
6015 }
6016
6017 void CodeGenerator::cgLdResumableArObj(IRInstruction* inst) {
6018   auto const dstReg = dstLoc(0).reg();
6019   auto const resumableArReg = srcLoc(0).reg();
6020   auto const objectOff = Resumable::objectOff() - Resumable::arOff();
6021   m_as.lea (resumableArReg[objectOff], dstReg);
6022 }
6023
6024 void CodeGenerator::cgIterInit(IRInstruction* inst) {
6025   cgIterInitCommon(inst);
6026 }
6027
6028 void CodeGenerator::cgIterInitK(IRInstruction* inst) {
6029   cgIterInitCommon(inst);
6030 }
6031
6032 void CodeGenerator::cgWIterInit(IRInstruction* inst) {
6033   cgIterInitCommon(inst);
6034 }
6035
6036 void CodeGenerator::cgWIterInitK(IRInstruction* inst) {
6037   cgIterInitCommon(inst);
6038 }
6039
6040 void CodeGenerator::cgIterInitCommon(IRInstruction* inst) {
6041   bool isInitK = inst->op() == IterInitK || inst->op() == WIterInitK;
6042   bool isWInit = inst->op() == WIterInit || inst->op() == WIterInitK;
6043
6044   PhysReg        fpReg = srcLoc(1).reg();
6045   int       iterOffset = this->iterOffset(inst->extra<IterData>()->iterId);
6046   int   valLocalOffset = localOffset(inst->extra<IterData>()->valId);
6047   SSATmp*          src = inst->src(0);
6048   auto args = argGroup();
6049   args.addr(fpReg, iterOffset).ssa(0/*src*/);
6050   if (src->isA(Type::Arr)) {
6051     args.addr(fpReg, valLocalOffset);
6052     if (isInitK) {
6053       args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6054     } else if (isWInit) {
6055       args.imm(0);
6056     }
6057     TCA helperAddr = isWInit ? (TCA)new_iter_array_key<true> :
6058       isInitK ? (TCA)new_iter_array_key<false> : (TCA)new_iter_array;
6059     cgCallHelper(
6060       m_as,
6061       CppCall::direct(reinterpret_cast<void (*)()>(helperAddr)),
6062       callDest(inst),
6063       SyncOptions::kSyncPoint,
6064       args);
6065   } else {
6066     assert(src->type() <= Type::Obj);
6067     args.imm(uintptr_t(curClass())).addr(fpReg, valLocalOffset);
6068     if (isInitK) {
6069       args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6070     } else {
6071       args.imm(0);
6072     }
6073     // new_iter_object decrefs its src object if it propagates an
6074     // exception out, so we use kSyncPointAdjustOne, which adjusts the
6075     // stack pointer by 1 stack element on an unwind, skipping over
6076     // the src object.
6077     cgCallHelper(m_as, CppCall::direct(new_iter_object), callDest(inst),
6078                  SyncOptions::kSyncPointAdjustOne, args);
6079   }
6080 }
6081
6082 void CodeGenerator::cgMIterInit(IRInstruction* inst) {
6083   cgMIterInitCommon(inst);
6084 }
6085
6086 void CodeGenerator::cgMIterInitK(IRInstruction* inst) {
6087   cgMIterInitCommon(inst);
6088 }
6089
6090 void CodeGenerator::cgMIterInitCommon(IRInstruction* inst) {
6091   PhysReg          fpReg = srcLoc(1).reg();
6092   int         iterOffset = this->iterOffset(inst->extra<IterData>()->iterId);
6093   int     valLocalOffset = localOffset(inst->extra<IterData>()->valId);
6094   SSATmp*            src = inst->src(0);
6095
6096   auto args = argGroup();
6097   args.addr(fpReg, iterOffset).ssa(0/*src*/);
6098
6099   assert(src->type().isBoxed());
6100   auto innerType = src->type().innerType();
6101   assert(innerType.isKnownDataType());
6102
6103   if (innerType <= Type::Arr) {
6104     args.addr(fpReg, valLocalOffset);
6105     if (inst->op() == MIterInitK) {
6106       args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6107     } else {
6108       args.imm(0);
6109     }
6110     cgCallHelper(m_as,
6111                  CppCall::direct(new_miter_array_key),
6112                  callDest(inst),
6113                  SyncOptions::kSyncPoint,
6114                  args);
6115   } else if (innerType <= Type::Obj) {
6116     args.immPtr(curClass()).addr(fpReg, valLocalOffset);
6117     if (inst->op() == MIterInitK) {
6118       args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6119     } else {
6120       args.imm(0);
6121     }
6122     // new_miter_object decrefs its src object if it propagates an
6123     // exception out, so we use kSyncPointAdjustOne, which adjusts the
6124     // stack pointer by 1 stack element on an unwind, skipping over
6125     // the src object.
6126     cgCallHelper(m_as,
6127                  CppCall::direct(new_miter_object),
6128                  callDest(inst),
6129                  SyncOptions::kSyncPointAdjustOne,
6130                  args);
6131   } else {
6132     CG_PUNT(MArrayIter-Unknown);
6133   }
6134 }
6135
6136 void CodeGenerator::cgIterNext(IRInstruction* inst) {
6137   cgIterNextCommon(inst);
6138 }
6139
6140 void CodeGenerator::cgIterNextK(IRInstruction* inst) {
6141   cgIterNextCommon(inst);
6142 }
6143
6144 void CodeGenerator::cgWIterNext(IRInstruction* inst) {
6145   cgIterNextCommon(inst);
6146 }
6147
6148 void CodeGenerator::cgWIterNextK(IRInstruction* inst) {
6149   cgIterNextCommon(inst);
6150 }
6151
6152 void CodeGenerator::cgIterNextCommon(IRInstruction* inst) {
6153   bool isNextK = inst->op() == IterNextK || inst->op() == WIterNextK;
6154   bool isWNext = inst->op() == WIterNext || inst->op() == WIterNextK;
6155   PhysReg fpReg = srcLoc(0).reg();
6156   auto args = argGroup();
6157   args.addr(fpReg, iterOffset(inst->extra<IterData>()->iterId))
6158       .addr(fpReg, localOffset(inst->extra<IterData>()->valId));
6159   if (isNextK) {
6160     args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6161   } else if (isWNext) {
6162     // We punt this case because nothing is using WIterNext opcodes
6163     // right now, and we don't want the witer_next_key helper to need
6164     // to check for null.
6165     CG_PUNT(WIterNext-nonKey);
6166   }
6167   TCA helperAddr = isWNext ? (TCA)witer_next_key :
6168     isNextK ? (TCA)iter_next_key_ind : (TCA)iter_next_ind;
6169   cgCallHelper(m_as,
6170                CppCall::direct(reinterpret_cast<void (*)()>(helperAddr)),
6171                callDest(inst),
6172                SyncOptions::kSyncPoint,
6173                args);
6174 }
6175
6176 void CodeGenerator::cgMIterNext(IRInstruction* inst) {
6177   cgMIterNextCommon(inst);
6178 }
6179
6180 void CodeGenerator::cgMIterNextK(IRInstruction* inst) {
6181   cgMIterNextCommon(inst);
6182 }
6183
6184 void CodeGenerator::cgMIterNextCommon(IRInstruction* inst) {
6185   PhysReg fpReg = srcLoc(0).reg();
6186   auto args = argGroup();
6187   args.addr(fpReg, iterOffset(inst->extra<IterData>()->iterId))
6188       .addr(fpReg, localOffset(inst->extra<IterData>()->valId));
6189   if (inst->op() == MIterNextK) {
6190     args.addr(fpReg, localOffset(inst->extra<IterData>()->keyId));
6191   } else {
6192     args.imm(0);
6193   }
6194   cgCallHelper(m_as, CppCall::direct(miter_next_key), callDest(inst),
6195                SyncOptions::kSyncPoint, args);
6196 }
6197
6198 void CodeGenerator::cgIterFree(IRInstruction* inst) {
6199   PhysReg fpReg = srcLoc(0).reg();
6200   int     offset = iterOffset(inst->extra<IterFree>()->iterId);
6201   cgCallHelper(m_as,
6202                CppCall::method(&Iter::free),
6203                kVoidDest,
6204                SyncOptions::kSyncPoint,
6205                argGroup().addr(fpReg, offset));
6206 }
6207
6208 void CodeGenerator::cgMIterFree(IRInstruction* inst) {
6209   PhysReg fpReg = srcLoc(0).reg();
6210   int    offset = iterOffset(inst->extra<MIterFree>()->iterId);
6211   cgCallHelper(m_as,
6212                CppCall::method(&Iter::mfree),
6213                kVoidDest,
6214                SyncOptions::kSyncPoint,
6215                argGroup().addr(fpReg, offset));
6216 }
6217
6218 void CodeGenerator::cgDecodeCufIter(IRInstruction* inst) {
6219   PhysReg fpReg = srcLoc(1).reg();
6220   int     offset = iterOffset(inst->extra<DecodeCufIter>()->iterId);
6221   cgCallHelper(m_as,
6222                CppCall::direct(decodeCufIterHelper),
6223                callDest(inst),
6224                SyncOptions::kSyncPoint,
6225                argGroup().addr(fpReg, offset)
6226                                   .typedValue(0));
6227 }
6228
6229 void CodeGenerator::cgCIterFree(IRInstruction* inst) {
6230   PhysReg fpReg = srcLoc(0).reg();
6231   int      offset = iterOffset(inst->extra<CIterFree>()->iterId);
6232   cgCallHelper(m_as,
6233                CppCall::method(&Iter::cfree),
6234                kVoidDest,
6235                SyncOptions::kSyncPoint,
6236                argGroup().addr(fpReg, offset));
6237 }
6238
6239 void CodeGenerator::cgNewStructArray(IRInstruction* inst) {
6240   auto data = inst->extra<NewStructData>();
6241   StringData** table = mcg->allocData<StringData*>(sizeof(StringData*),
6242                                                       data->numKeys);
6243   memcpy(table, data->keys, data->numKeys * sizeof(*data->keys));
6244   MixedArray* (*f)(uint32_t, StringData**, const TypedValue*) =
6245     &MixedArray::MakeStruct;
6246   cgCallHelper(m_as,
6247                CppCall::direct(f),
6248                callDest(inst),
6249                SyncOptions::kNoSyncPoint,
6250                argGroup().imm(data->numKeys)
6251                          .imm(uintptr_t(table))
6252                          .ssa(0/*values*/)
6253   );
6254 }
6255
6256 void CodeGenerator::cgIncStat(IRInstruction *inst) {
6257   emitIncStat(m_mainCode,
6258               Stats::StatCounter(inst->src(0)->intVal()),
6259               inst->src(1)->intVal(),
6260               inst->src(2)->boolVal());
6261 }
6262
6263 void CodeGenerator::cgIncTransCounter(IRInstruction* inst) {
6264   emitTransCounterInc(m_as);
6265 }
6266
6267 void CodeGenerator::cgIncProfCounter(IRInstruction* inst) {
6268   TransID  transId = inst->extra<TransIDData>()->transId;
6269   auto counterAddr = mcg->tx().profData()->transCounterAddr(transId);
6270   emitLoadImm(m_as, uint64_t(counterAddr), m_rScratch);
6271   m_as.decq(m_rScratch[0]);
6272 }
6273
6274 void CodeGenerator::cgDbgAssertRefCount(IRInstruction* inst) {
6275   emitAssertRefCount(m_as, srcLoc(0).reg());
6276 }
6277
6278 void CodeGenerator::cgDbgAssertType(IRInstruction* inst) {
6279   emitTypeTest(inst->typeParam(),
6280                srcLoc(0).reg(1),
6281                srcLoc(0).reg(0),
6282                [&](ConditionCode cc) {
6283                  ifThen(m_as, ccNegate(cc), [&](Asm& a) { a.ud2(); });
6284                });
6285 }
6286
6287 /*
6288  * Defined in translator-asm-helpers.S. Used for an assert in DbgAssertRetAddr.
6289  */
6290 extern "C" void enterTCServiceReq();
6291
6292 void CodeGenerator::cgDbgAssertRetAddr(IRInstruction* inst) {
6293   // With the exception of FreeActRec and RetCtrl, the native return address
6294   // should always be the part of enterTCHelper that handles service
6295   // requests. To keep things reasonable we only emit this at the beginning of
6296   // a bytecode's translation, which should never begin with FreeActRec or
6297   // RetCtrl.
6298   always_assert(!inst->is(FreeActRec, RetCtrl));
6299   auto v = vmain();
6300   Immed64 imm = (uintptr_t)enterTCServiceReq;
6301   if (imm.fits(sz::dword)) {
6302     v << cmpqim{imm.l(), *rsp};
6303   } else {
6304     v << ldimm{imm, m_rScratch};
6305     v << cmpqm{m_rScratch, *rsp};
6306   }
6307   ifThen(v, CC_NE, [&](Vout& v) {
6308      v << ud2{};
6309   });
6310 }
6311
6312 void CodeGenerator::emitVerifyCls(IRInstruction* inst) {
6313   auto const objClass = inst->src(0);
6314   auto const objClassReg = srcLoc(0).reg();
6315   auto const constraint = inst->src(1);
6316   auto const constraintReg = srcLoc(1).reg();
6317
6318   auto& v = vmain();
6319   if (constraintReg == InvalidReg) {
6320     if (objClassReg != InvalidReg) {
6321       auto constraintCls = constraint->clsVal();
6322       auto constraintImm = Immed64(constraintCls);
6323       if (constraintImm.fits(sz::dword)) {
6324         v << cmpqi{constraintImm.l(), objClassReg};
6325       } else {
6326         auto constraintTmp = v.makeReg();
6327         v << ldimm{constraintCls, constraintTmp};
6328         v << cmpq{constraintTmp, objClassReg};
6329       }
6330     } else {
6331       // Both constant.
6332       if (objClass->clsVal() == constraint->clsVal()) return;
6333       return cgCallNative(v, inst);
6334     }
6335   } else if (objClassReg != InvalidReg) {
6336     v << cmpq{constraintReg, objClassReg};
6337   } else {
6338     // Reverse the args because cmpq can only have a constant in the LHS.
6339     auto objCls = objClass->clsVal();
6340     auto objImm = Immed64(objCls);
6341     if (objImm.fits(sz::dword)) {
6342       v << cmpqi{objImm.l(), constraintReg};
6343     } else {
6344       auto objTmp = v.makeReg();
6345       v << ldimm{objCls, objTmp};
6346       v << cmpq{objTmp, constraintReg};
6347     }
6348   }
6349
6350   // The native call for this instruction is the slow path that does
6351   // proper subtype checking. The comparison above is just to
6352   // short-circuit the overhead when the Classes are an exact match.
6353   ifThen(v, CC_NE, [&](Vout& v){ cgCallNative(v, inst); });
6354 }
6355
6356 void CodeGenerator::cgVerifyParamCls(IRInstruction* inst) {
6357   emitVerifyCls(inst);
6358 }
6359
6360 void CodeGenerator::cgVerifyRetCls(IRInstruction* inst) {
6361   emitVerifyCls(inst);
6362 }
6363
6364 void CodeGenerator::cgRBTrace(IRInstruction* inst) {
6365   auto const& extra = *inst->extra<RBTrace>();
6366
6367   TCA helper;
6368   auto args = argGroup();
6369   if (extra.msg) {
6370     auto const msg = extra.msg;
6371     assert(msg->isStatic());
6372     args.immPtr(msg->data());
6373     args.imm(msg->size());
6374     args.imm(extra.type);
6375     helper = (TCA)Trace::ringbufferMsg;
6376   } else {
6377     auto const sk = extra.sk;
6378     args.imm(extra.type);
6379     args.imm(sk.toAtomicInt());
6380     args.ipRel(m_as.frontier());
6381     helper = (TCA)Trace::ringbufferEntry;
6382   }
6383
6384   cgCallHelper(m_as,
6385                CppCall::direct(reinterpret_cast<void (*)()>(helper)),
6386                kVoidDest,
6387                SyncOptions::kNoSyncPoint,
6388                args);
6389 }
6390
6391 void CodeGenerator::cgLdClsInitData(IRInstruction* inst) {
6392   auto clsReg = srcLoc(0).reg();
6393   auto dstReg = dstLoc(0).reg();
6394   auto offset = Class::propDataCacheOff() +
6395                 RDS::Link<Class::PropInitVec*>::handleOff();
6396   m_as.  loadl(clsReg[offset], r32(dstReg));
6397   m_as.  loadq(rVmTl[dstReg], dstReg);
6398   m_as.  loadq(dstReg[Class::PropInitVec::dataOff()], dstReg);
6399 }
6400
6401 void CodeGenerator::cgConjure(IRInstruction* inst) {
6402   m_as.ud2();
6403 }
6404
6405 void CodeGenerator::cgProfileStr(IRInstruction* inst) {
6406   TargetProfile<StrProfile> profile(m_unit.context(), inst->marker(),
6407                                     inst->extra<ProfileStrData>()->key);
6408   assert(profile.profiling());
6409   auto const ch = profile.handle();
6410
6411   auto ptrReg = srcLoc(0).reg();
6412   emitCmpTVType(m_as, KindOfStaticString, ptrReg[TVOFF(m_type)]);
6413   ifThenElse(
6414     m_as, CC_E,
6415     [&](Asm& a) { // m_type == KindOfStaticString
6416       a.incl(rVmTl[ch + offsetof(StrProfile, staticStr)]);
6417     },
6418     [&](Asm& a) { // m_type == KindOfString
6419       a.loadq(ptrReg[TVOFF(m_data)], m_rScratch);
6420       a.cmpl(StaticValue, m_rScratch[FAST_REFCOUNT_OFFSET]);
6421
6422       ifThenElse(
6423         a, CC_E,
6424         [&](Asm& a) { // _count == StaticValue
6425           a.incl(rVmTl[ch + offsetof(StrProfile, strStatic)]);
6426         },
6427         [&](Asm& a) {
6428           a.incl(rVmTl[ch + offsetof(StrProfile, str)]);
6429         }
6430       );
6431     }
6432   );
6433 }
6434
6435 void CodeGenerator::cgCountArray(IRInstruction* inst) {
6436   auto const baseReg = srcLoc(0).reg();
6437   auto const dstReg  = dstLoc(0).reg();
6438   auto& v = vmain();
6439
6440   v << cmpbim{ArrayData::kNvtwKind, baseReg[ArrayData::offsetofKind()]};
6441   unlikelyIfThenElse(v, vcold(), CC_Z,
6442     [&](Vout& v) {
6443       cgCallNative(v, inst);
6444     },
6445     [&](Vout& v) {
6446       v << loadl{baseReg[ArrayData::offsetofSize()], dstReg};
6447     }
6448   );
6449 }
6450
6451 void CodeGenerator::cgCountArrayFast(IRInstruction* inst) {
6452   auto const baseReg = srcLoc(0).reg();
6453   auto const dstReg  = dstLoc(0).reg();
6454
6455   m_as. loadl(baseReg[ArrayData::offsetofSize()], r32(dstReg));
6456 }
6457
6458 void CodeGenerator::cgCountCollection(IRInstruction* inst) {
6459   auto const baseReg = srcLoc(0).reg();
6460   auto const dstReg  = dstLoc(0).reg();
6461
6462   m_as. loadl(baseReg[FAST_COLLECTION_SIZE_OFFSET], r32(dstReg));
6463 }
6464
6465 void CodeGenerator::print() const {
6466   JIT::print(std::cout, m_unit, &m_state.regs, m_state.asmInfo);
6467 }
6468
6469 }}}