Better reffiness checks

[hiphop-php.git] / hphp / runtime / vm / jit / codegen.cpp

/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/

#include "hphp/runtime/vm/jit/codegen.h"

#include <cstring>
#include <unwind.h>

#include "folly/ScopeGuard.h"
#include "folly/Format.h"
#include "hphp/util/trace.h"
#include "hphp/util/util.h"

#include "hphp/runtime/base/array/hphp_array.h"
#include "hphp/runtime/base/comparisons.h"
#include "hphp/runtime/base/complex_types.h"
#include "hphp/runtime/base/runtime_option.h"
#include "hphp/runtime/base/string_data.h"
#include "hphp/runtime/base/types.h"
#include "hphp/runtime/ext/ext_closure.h"
#include "hphp/runtime/ext/ext_continuation.h"
#include "hphp/runtime/ext/ext_collections.h"
#include "hphp/runtime/vm/bytecode.h"
#include "hphp/runtime/vm/runtime.h"
#include "hphp/runtime/base/stats.h"
#include "hphp/runtime/vm/jit/targetcache.h"
#include "hphp/runtime/vm/jit/translator-inline.h"
#include "hphp/runtime/vm/jit/translator-x64.h"
#include "hphp/runtime/vm/jit/translator-x64-internal.h"
#include "hphp/runtime/vm/jit/translator.h"
#include "hphp/runtime/vm/jit/types.h"
#include "hphp/runtime/vm/jit/x64-util.h"
#include "hphp/runtime/vm/jit/ir.h"
#include "hphp/runtime/vm/jit/linearscan.h"
#include "hphp/runtime/vm/jit/nativecalls.h"
#include "hphp/runtime/vm/jit/print.h"
#include "hphp/runtime/vm/jit/layout.h"

using HPHP::Transl::TCA;
using namespace HPHP::Transl::TargetCache;

namespace HPHP {
namespace JIT {

namespace {

//////////////////////////////////////////////////////////////////////

using namespace Util;
using namespace Transl::reg;

TRACE_SET_MOD(hhir);

/*
 * It's not normally ok to directly use tracelet abi registers in
 * codegen, unless you're directly dealing with an instruction that
 * does near-end-of-tracelet glue.  (Or also we sometimes use them
 * just for some static_assertions relating to calls to helpers from
 * tx64 that hardcode these registers.)
 */
using Transl::rVmFp;
using Transl::rVmSp;

const size_t kTypeWordOffset = (offsetof(TypedValue, m_type) % 8);
const size_t kTypeShiftBits = kTypeWordOffset * CHAR_BIT;

// left shift an immediate DataType, for type, to the correct position
// within one of the registers used to pass a TypedValue by value.
uint64_t toDataTypeForCall(Type type) {
  return uint64_t(type.toDataType()) << kTypeShiftBits;
}

int64_t spillSlotsToSize(int n) {
  return n * sizeof(int64_t);
}

void cgPunt(const char* file, int line, const char* func, uint32_t bcOff) {
  if (dumpIREnabled()) {
    HPHP::Trace::trace("--------- CG_PUNT %s %d %s  bcOff: %d \n",
                       file, line, func, bcOff);
  }
  throw FailedCodeGen(file, line, func, bcOff);
}

#define CG_PUNT(instr) cgPunt(__FILE__, __LINE__, #instr, m_curBcOff)

struct CycleInfo {
  int node;
  int length;
};

struct MoveInfo {
  enum Kind { Move, Xchg };

  MoveInfo(Kind kind, int reg1, int reg2):
      m_kind(kind), m_reg1(reg1), m_reg2(reg2) {}

  Kind m_kind;
  PhysReg m_reg1, m_reg2;
};

template <int N>
static bool cycleHasXMMReg(const CycleInfo& cycle,
                           const int (&moves)[N]) {
  int first = cycle.node;
  int node = first;
  do {
    if (PhysReg(node).isXMM()) return true;
    node = moves[node];
  } while (node != first);
  return false;
}

template <int N>
void doRegMoves(int (&moves)[N], int rTmp,
                std::vector<MoveInfo>& howTo) {
  assert(howTo.empty());
  int outDegree[N];
  CycleInfo cycles[N];
  int numCycles = 0;
  // Iterate over the nodes filling in outDegree[] and cycles[] as we go
  {
    int index[N];
    for (int node = 0; node < N; ++node) {
      // If a node's source is itself, its a nop
      if (moves[node] == node) moves[node] = -1;
      if (node == rTmp && moves[node] >= 0) {
        // ERROR: rTmp cannot be referenced in moves[].
        assert(false);
      }
      outDegree[node] = 0;
      index[node] = -1;
    }
    int nextIndex = 0;
    for (int startNode = 0; startNode < N; ++startNode) {
      // If startNode has not been visited yet, begin walking
      // a path from start node
      if (index[startNode] < 0) {
        int node = startNode;
pathloop:
        index[node] = nextIndex++;
        if (moves[node] >= 0) {
          int nextNode = moves[node];
          ++outDegree[nextNode];
          if (index[nextNode] < 0) {
            // If there is an edge from v to nextNode and nextNode has not been
            // visited, extend the current path to include nextNode and recurse
            node = nextNode;
            goto pathloop;
          }
          // There is an edge from v to nextNode but nextNode has already been
          // visited, check if nextNode is on the current path
          if (index[nextNode] >= index[startNode]) {
            // nextNode is on the current path so we've found a cycle
            int length = nextIndex - index[nextNode];
            CycleInfo ci = { nextNode, length };
            cycles[numCycles] = ci;
            ++numCycles;
          }
        }
      }
    }
  }
  // Handle all moves that aren't part of a cycle
  {
    int q[N];
    int qBack = 0;
    for (int node = 0; node < N; ++node) {
      if (outDegree[node] == 0) {
        q[qBack] = node;
        ++qBack;
      }
    }
    for (int i = 0; i < qBack; ++i) {
      int node = q[i];
      if (moves[node] >= 0) {
        int nextNode = moves[node];
        howTo.push_back(MoveInfo(MoveInfo::Move, nextNode, node));
        --outDegree[nextNode];
        if (outDegree[nextNode] == 0) {
          q[qBack] = nextNode;
          ++qBack;
        }
      }
    }
  }
  // Deal with any cycles we encountered
  for (int i = 0; i < numCycles; ++i) {
    // can't use xchg if one of the registers is XMM
    bool hasXMMReg = cycleHasXMMReg(cycles[i], moves);
    if (cycles[i].length == 2 && !hasXMMReg) {
      int v = cycles[i].node;
      int w = moves[v];
      howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
    } else if (cycles[i].length == 3 && !hasXMMReg) {
      int v = cycles[i].node;
      int w = moves[v];
      howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
      int x = moves[w];
      howTo.push_back(MoveInfo(MoveInfo::Xchg, x, w));
    } else {
      int v = cycles[i].node;
      howTo.push_back(MoveInfo(MoveInfo::Move, v, rTmp));
      int w = v;
      int x = moves[w];
      while (x != v) {
        howTo.push_back(MoveInfo(MoveInfo::Move, x, w));
        w = x;
        x = moves[w];
      }
      howTo.push_back(MoveInfo(MoveInfo::Move, rTmp, w));
    }
  }
}

const char* getContextName(Class* ctx) {
  return ctx ? ctx->name()->data() : ":anonymous:";
}

} // unnamed namespace

//////////////////////////////////////////////////////////////////////

ArgDesc::ArgDesc(SSATmp* tmp, const RegisterInfo& info, bool val)
  : m_imm(-1), m_zeroExtend(false), m_done(false) {
  if (tmp->type() == Type::None) {
    assert(val);
    m_kind = None;
    return;
  }
  if (tmp->inst()->op() == DefConst) {
    m_srcReg = InvalidReg;
    if (val) {
      m_imm = tmp->getValBits();
    } else {
      m_imm = toDataTypeForCall(tmp->type());
    }
    m_kind = Imm;
    return;
  }
  if (tmp->type().isNull()) {
    m_srcReg = InvalidReg;
    if (val) {
      m_imm = 0;
    } else {
      m_imm = toDataTypeForCall(tmp->type());
    }
    m_kind = Imm;
    return;
  }
  if (val || tmp->numNeededRegs() > 1) {
    auto reg = info.reg(val ? 0 : 1);
    assert(reg != InvalidReg);
    m_imm = 0;

    // If val is false then we're passing tmp's type. TypeReg lets
    // CodeGenerator know that the value might require some massaging
    // to be in the right format for the call.
    m_kind = val ? Reg : TypeReg;
    // zero extend any boolean value that we pass to the helper in case
    // the helper expects it (e.g., as TypedValue)
    if (val && tmp->isA(Type::Bool)) m_zeroExtend = true;
    m_srcReg = reg;
    return;
  }
  m_srcReg = InvalidReg;
  m_imm = toDataTypeForCall(tmp->type());
  m_kind = Imm;
}

const Func* CodeGenerator::curFunc() const {
  always_assert(m_state.lastMarker &&
                "We shouldn't be looking for a func when we have no marker");
  return m_state.lastMarker->func;
}

/*
 * Select a scratch register to use in the given instruction, prefering the
 * lower registers which don't require a REX prefix.  The selected register
 * must not be any of the instructions inputs or outputs, and neither a register
 * that is alive across this instruction.
 */
PhysReg CodeGenerator::selectScratchReg(IRInstruction* inst) {
  static const RegSet kLowGPRegs = RegSet()
    | RegSet(reg::rax)
    | RegSet(reg::rcx)
    | RegSet(reg::rdx)
    | RegSet(reg::rsi)
    | RegSet(reg::rdi)
    ;
  RegSet liveRegs = m_state.liveRegs[inst];
  for (const auto& tmp : inst->srcs()) {
    liveRegs |= m_regs[tmp].regs();
  }
  for (const auto& tmp : inst->dsts()) {
    liveRegs |= m_regs[tmp].regs();
  }
  PhysReg selectedReg;
  if ((kLowGPRegs - liveRegs).findFirst(selectedReg)) {
    return selectedReg;
  }
  return rCgGP;
}

Address CodeGenerator::cgInst(IRInstruction* inst) {
  Opcode opc = inst->op();
  auto const start = m_as.code.frontier;
  m_rScratch = selectScratchReg(inst);
  if (inst->taken() && inst->taken()->trace()->isCatch()) {
    m_state.catchTrace = inst->taken()->trace();
  } else {
    m_state.catchTrace = nullptr;
  }

  switch (opc) {
#define O(name, dsts, srcs, flags)                                \
  case name: FTRACE(7, "cg" #name "\n");                          \
             cg ## name (inst);                                   \
             return m_as.code.frontier == start ? nullptr : start;
  IR_OPCODES
#undef O

  default:
    assert(0);
    return nullptr;
  }
}

#define NOOP_OPCODE(opcode) \
  void CodeGenerator::cg##opcode(IRInstruction*) {}

#define CALL_OPCODE(opcode) \
  void CodeGenerator::cg##opcode(IRInstruction* i) { cgCallNative(i); }

#define CALL_STK_OPCODE(opcode) \
  CALL_OPCODE(opcode)           \
  CALL_OPCODE(opcode ## Stk)

NOOP_OPCODE(DefConst)
NOOP_OPCODE(DefFP)
NOOP_OPCODE(DefSP)
NOOP_OPCODE(AssertLoc)
NOOP_OPCODE(OverrideLoc)
NOOP_OPCODE(AssertStk)
NOOP_OPCODE(Nop)
NOOP_OPCODE(DefLabel)
NOOP_OPCODE(ExceptionBarrier)

CALL_OPCODE(AddElemStrKey)
CALL_OPCODE(AddElemIntKey)
CALL_OPCODE(AddNewElem)
CALL_OPCODE(ArrayAdd)
CALL_OPCODE(Box)

CALL_OPCODE(ConvBoolToArr);
CALL_OPCODE(ConvDblToArr);
CALL_OPCODE(ConvIntToArr);
CALL_OPCODE(ConvObjToArr);
CALL_OPCODE(ConvStrToArr);
CALL_OPCODE(ConvCellToArr);

CALL_OPCODE(ConvArrToBool);
CALL_OPCODE(ConvStrToBool);
CALL_OPCODE(ConvCellToBool);

CALL_OPCODE(ConvArrToDbl);
CALL_OPCODE(ConvObjToDbl);
CALL_OPCODE(ConvStrToDbl);
CALL_OPCODE(ConvCellToDbl);

CALL_OPCODE(ConvArrToInt);
CALL_OPCODE(ConvDblToInt);
CALL_OPCODE(ConvObjToInt);
CALL_OPCODE(ConvStrToInt);
CALL_OPCODE(ConvCellToInt);

CALL_OPCODE(ConvCellToObj);

CALL_OPCODE(ConvDblToStr);
CALL_OPCODE(ConvIntToStr);
CALL_OPCODE(ConvObjToStr);
CALL_OPCODE(ConvCellToStr);

CALL_OPCODE(CreateCont)
CALL_OPCODE(FillContLocals)
CALL_OPCODE(NewArray)
CALL_OPCODE(NewTuple)
CALL_OPCODE(AllocObj)
CALL_OPCODE(LdClsCtor);
CALL_OPCODE(CreateCl)
CALL_OPCODE(PrintStr)
CALL_OPCODE(PrintInt)
CALL_OPCODE(PrintBool)
CALL_OPCODE(DbgAssertPtr)
CALL_OPCODE(LdSwitchDblIndex)
CALL_OPCODE(LdSwitchStrIndex)
CALL_OPCODE(LdSwitchObjIndex)
CALL_OPCODE(VerifyParamCallable)
CALL_OPCODE(VerifyParamFail)
CALL_OPCODE(RaiseUninitLoc)
CALL_OPCODE(WarnNonObjProp)
CALL_OPCODE(ThrowNonObjProp)
CALL_OPCODE(RaiseUndefProp)
CALL_OPCODE(RaiseError)
CALL_OPCODE(RaiseWarning)
CALL_OPCODE(IncStatGrouped)
CALL_OPCODE(StaticLocInit)
CALL_OPCODE(StaticLocInitCached)
CALL_OPCODE(OpMod)
CALL_OPCODE(ArrayIdx)

// Vector instruction helpers
CALL_OPCODE(BaseG)
CALL_OPCODE(PropX)
CALL_STK_OPCODE(PropDX)
CALL_OPCODE(CGetProp)
CALL_STK_OPCODE(VGetProp)
CALL_STK_OPCODE(BindProp)
CALL_STK_OPCODE(SetProp)
CALL_OPCODE(UnsetProp)
CALL_STK_OPCODE(SetOpProp)
CALL_STK_OPCODE(IncDecProp)
CALL_OPCODE(EmptyProp)
CALL_OPCODE(IssetProp)
CALL_OPCODE(ElemX)
CALL_STK_OPCODE(ElemDX)
CALL_STK_OPCODE(ElemUX)
CALL_OPCODE(ArrayGet)
CALL_OPCODE(VectorGet)
CALL_OPCODE(MapGet)
CALL_OPCODE(CGetElem)
CALL_STK_OPCODE(VGetElem)
CALL_STK_OPCODE(BindElem)
CALL_STK_OPCODE(SetWithRefElem)
CALL_STK_OPCODE(SetWithRefNewElem)
CALL_OPCODE(ArraySet)
CALL_OPCODE(VectorSet)
CALL_OPCODE(MapSet)
CALL_OPCODE(ArraySetRef)
CALL_STK_OPCODE(SetElem)
CALL_STK_OPCODE(UnsetElem)
CALL_STK_OPCODE(SetOpElem)
CALL_STK_OPCODE(IncDecElem)
CALL_STK_OPCODE(SetNewElem)
CALL_STK_OPCODE(BindNewElem)
CALL_OPCODE(ArrayIsset)
CALL_OPCODE(VectorIsset)
CALL_OPCODE(MapIsset)
CALL_OPCODE(IssetElem)
CALL_OPCODE(EmptyElem)

#undef NOOP_OPCODE

// Thread chain of patch locations using the 4 byte space in each jmp/jcc
static void prependPatchAddr(CodegenState& state,
                             Block* block,
                             TCA patchAddr) {
  auto &patches = state.patches;
  ssize_t diff = patches[block] ? (patchAddr - (TCA)patches[block]) : 0;
  assert(deltaFits(diff, sz::dword));
  *(int32_t*)(patchAddr) = (int32_t)diff;
  patches[block] = patchAddr;
}

static void emitFwdJmp(Asm& a, Block* target, CodegenState& state) {
  if (auto addr = state.addresses[target]) {
    return a.jmpAuto(addr);
  }

  // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
  a.jmp(a.code.frontier);
  TCA immPtr = a.code.frontier - 4;
  prependPatchAddr(state, target, immPtr);
}

void CodeGenerator::emitFwdJcc(Asm& a, ConditionCode cc, Block* target) {
  if (auto addr = m_state.addresses[target]) {
    return a.jccAuto(cc, addr);
  }

  // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
  a.jcc(cc, a.code.frontier);
  TCA immPtr = a.code.frontier - 4;
  prependPatchAddr(m_state, target, immPtr);
}

void CodeGenerator::emitFwdJcc(ConditionCode cc, Block* target) {
  emitFwdJcc(m_as, cc, target);
}

void emitLoadImm(CodeGenerator::Asm& as, int64_t val, PhysReg dstReg) {
  as.emitImmReg(val, dstReg);
}

static void
emitMovRegReg(CodeGenerator::Asm& as, PhysReg srcReg, PhysReg dstReg) {
  assert(srcReg != InvalidReg);
  assert(dstReg != InvalidReg);

  if (srcReg == dstReg) return;

  if (srcReg.isGP()) {
    if (dstReg.isGP()) {                 // GP => GP
      as.movq(srcReg, dstReg);
    } else {                             // GP => XMM
      // This generates a movq x86 instruction, which zero extends
      // the 64-bit value in srcReg into a 128-bit XMM register
      as.mov_reg64_xmm(srcReg, dstReg);
    }
  } else {
    if (dstReg.isGP()) {                 // XMM => GP
      as.mov_xmm_reg64(srcReg, dstReg);
    } else {                             // XMM => XMM
      // This copies all 128 bits in XMM,
      // thus avoiding partial register stalls
      as.movdqa(srcReg, dstReg);
    }
  }
}

void CodeGenerator::emitLoadImm(CodeGenerator::Asm& as, int64_t val,
                                PhysReg dstReg) {
  assert(dstReg != InvalidReg);
  if (dstReg.isGP()) {
    as.emitImmReg(val, dstReg);
  } else {
    assert(dstReg.isXMM());
    if (val == 0) {
      as.pxor_xmm_xmm(dstReg, dstReg);
    } else {
      // Can't move immediate directly into XMM register, so use m_rScratch
      as.emitImmReg(val, m_rScratch);
      emitMovRegReg(as, m_rScratch, dstReg);
    }
  }
}

static void emitLea(CodeGenerator::Asm& as, MemoryRef mr, PhysReg dst) {
  if (dst == InvalidReg) return;
  if (mr.r.disp == 0) {
    emitMovRegReg(as, mr.r.base, dst);
  } else {
    as.lea(mr, dst);
  }
}

template<class Mem>
static void emitLoadReg(CodeGenerator::Asm& as, Mem mem, PhysReg reg) {
  assert(reg != InvalidReg);
  if (reg.isGP()) {
    as.loadq(mem, reg);
  } else {
    as.movsd(mem, reg);
  }
}

template<class Mem>
static void emitStoreReg(CodeGenerator::Asm& as, PhysReg reg, Mem mem) {
  assert(reg != InvalidReg);
  if (reg.isGP()) {
    as.storeq(reg, mem);
  } else {
    as.movsd(reg, mem);
  }
}

static void shuffle2(CodeGenerator::Asm& a,
                     PhysReg s0, PhysReg s1, PhysReg d0, PhysReg d1) {
  assert(s0 != s1);
  if (d0 == s1 && d1 != InvalidReg) {
    assert(d0 != d1);
    if (d1 == s0) {
      a.    xchgq (s1, s0);
    } else {
      a.    movq (s1, d1); // save s1 first; d1 != s0
      a.    movq (s0, d0);
    }
  } else {
    if (d0 != InvalidReg) emitMovRegReg(a, s0, d0); // d0 != s1
    if (d1 != InvalidReg) emitMovRegReg(a, s1, d1);
  }
}

static void zeroExtendBool(X64Assembler& as, const RegisterInfo& info) {
  auto reg = info.reg();
  if (reg != InvalidReg) {
    // zero-extend the bool from a byte to a quad
    // note: movzbl actually extends the value to 64 bits.
    as.movzbl(rbyte(reg), r32(reg));
  }
}

static void zeroExtendIfBool(X64Assembler& as, const SSATmp* src,
                            const RegisterInfo& info) {
  if (src->isA(Type::Bool)) {
    zeroExtendBool(as, info);
  }
}

static int64_t convIntToDouble(int64_t i) {
  union {
    double  d;
    int64_t i;
  } u;
  u.d = double(i);
  return u.i;
}

/*
 * Returns a XMM register containing the value of SSATmp tmp,
 * which can be either a bool, an int, or a double.
 * If the value is already in a XMM register, simply returns it.
 * Otherwise, the value is moved into rCgXMM, which is returned.
 * If instructions to convert to a double at runtime are needed,
 * they're emitted in 'as'.
 */
PhysReg CodeGenerator::prepXMMReg(const SSATmp* tmp,
                                  X64Assembler& as,
                                  const RegAllocInfo& allocInfo,
                                  RegXMM rCgXMM) {
  assert(tmp->isA(Type::Bool) || tmp->isA(Type::Int) || tmp->isA(Type::Dbl));

  PhysReg reg = allocInfo[tmp].reg();

  // Case 1: tmp is already in a XMM register
  if (reg.isXMM()) return reg;

  // Case 2: tmp is in a GP register
  if (reg != InvalidReg) {
    // Case 2.a: Dbl stored in GP reg
    if (tmp->isA(Type::Dbl)) {
      emitMovRegReg(as, reg, rCgXMM);
      return rCgXMM;
    }
    // Case 2.b: Bool or Int stored in GP reg
    assert(tmp->isA(Type::Bool) || tmp->isA(Type::Int));
    zeroExtendIfBool(as, tmp, allocInfo[tmp]);
    as.pxor_xmm_xmm(rCgXMM, rCgXMM);
    as.cvtsi2sd_reg64_xmm(reg, rCgXMM);
    return rCgXMM;
  }

  // Case 3: tmp is a constant
  assert(tmp->isConst());

  int64_t val = tmp->getValRawInt();
  if (!tmp->isA(Type::Dbl)) {
    assert(tmp->isA(Type::Bool | Type::Int));
    if (tmp->isA(Type::Bool)) val = val != 0;  // see task #2401790
    val = convIntToDouble(val);
  }
  emitLoadImm(as, val, m_rScratch);
  emitMovRegReg(as, m_rScratch, rCgXMM);
  return rCgXMM;
}

void CodeGenerator::doubleCmp(X64Assembler& a, RegXMM xmmReg0, RegXMM xmmReg1) {
  a.    ucomisd_xmm_xmm(xmmReg0, xmmReg1);
  Label notPF;
  a.    jnp8(notPF);
  // PF means the doubles were unordered. We treat this as !equal, so
  // clear ZF.
  a.    or_imm32_reg64(1, m_rScratch);
  asm_label(a, notPF);
}

static ConditionCode opToConditionCode(Opcode opc) {
  using namespace HPHP::Transl;

  switch (opc) {
  case JmpGt:                 return CC_G;
  case JmpGte:                return CC_GE;
  case JmpLt:                 return CC_L;
  case JmpLte:                return CC_LE;
  case JmpEq:                 return CC_E;
  case JmpNeq:                return CC_NE;
  case JmpSame:               return CC_E;
  case JmpNSame:              return CC_NE;
  case JmpInstanceOfBitmask:  return CC_NZ;
  case JmpNInstanceOfBitmask: return CC_Z;
  case JmpIsType:             return CC_NZ;
  case JmpIsNType:            return CC_Z;
  case JmpZero:               return CC_Z;
  case JmpNZero:              return CC_NZ;
  case ReqBindJmpGt:                 return CC_G;
  case ReqBindJmpGte:                return CC_GE;
  case ReqBindJmpLt:                 return CC_L;
  case ReqBindJmpLte:                return CC_LE;
  case ReqBindJmpEq:                 return CC_E;
  case ReqBindJmpNeq:                return CC_NE;
  case ReqBindJmpSame:               return CC_E;
  case ReqBindJmpNSame:              return CC_NE;
  case ReqBindJmpInstanceOfBitmask:  return CC_NZ;
  case ReqBindJmpNInstanceOfBitmask: return CC_Z;
  case ReqBindJmpZero:               return CC_Z;
  case ReqBindJmpNZero:              return CC_NZ;
  default:
    always_assert(0);
  }
}

void CodeGenerator::emitCompare(SSATmp* src1, SSATmp* src2) {
  auto const src1Type = src1->type();
  auto const src2Type = src2->type();

  // can't generate CMP instructions correctly for anything that isn't
  // a bool or a numeric, and we can't mix bool/numerics because
  // -1 == true in PHP, but not in HHIR binary representation
  if (!((src1Type == Type::Int && src2Type == Type::Int) ||
        ((src1Type == Type::Int || src1Type == Type::Dbl) &&
         (src2Type == Type::Int || src2Type == Type::Dbl)) ||
        (src1Type == Type::Bool && src2Type == Type::Bool) ||
        (src1Type == Type::Cls && src2Type == Type::Cls))) {
    CG_PUNT(emitCompare);
  }
  if (src1Type == Type::Dbl || src2Type == Type::Dbl) {
    PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, rCgXMM0);
    PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
    assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);
    doubleCmp(m_as, srcReg1, srcReg2);
  } else {
    auto srcReg1 = m_regs[src1].reg();
    auto srcReg2 = m_regs[src2].reg();

    // Note: when both src1 and src2 are constants, we should transform the
    // branch into an unconditional jump earlier in the IR.
    if (src1->isConst()) {
      // TODO: use compare with immediate or make sure simplifier
      // canonicalizes this so that constant is src2
      srcReg1 = m_rScratch;
      m_as.      mov_imm64_reg(src1->getValRawInt(), srcReg1);
    }
    if (src2->isConst()) {
      if (src1Type.subtypeOf(Type::Bool)) {
        m_as.    cmpb (src2->getValRawInt(), Reg8(int(srcReg1)));
      } else {
        m_as.    cmp_imm64_reg64(src2->getValRawInt(), srcReg1);
      }
    } else {
      // Note the reverse syntax in the assembler.
      // This cmp will compute srcReg1 - srcReg2
      if (src1Type.subtypeOf(Type::Bool)) {
        m_as.    cmpb (Reg8(int(srcReg2)), Reg8(int(srcReg1)));
      } else {
        m_as.    cmp_reg64_reg64(srcReg2, srcReg1);
      }
    }
  }
}

void CodeGenerator::emitReqBindJcc(ConditionCode cc,
                                   const ReqBindJccData* extra) {
  auto& a = m_as;
  assert(&m_as != &m_astubs &&
         "ReqBindJcc only makes sense outside of astubs");

  prepareForTestAndSmash(a, 0, kAlignJccAndJmp);
  auto const patchAddr = a.code.frontier;

  auto const jccStub = m_astubs.code.frontier;
  {
    auto& a = m_astubs;
    // TODO(#2404398): move the setcc into the generic stub code so we
    // don't need SRFlags::Persistent.
    a.    setcc  (cc, rbyte(serviceReqArgRegs[4]));
    m_tx64->emitServiceReq(
      SRFlags::Persistent,
      REQ_BIND_JMPCC_FIRST,
      4ull,
      patchAddr,
      uint64_t(extra->taken),
      uint64_t(extra->notTaken),
      uint64_t(cc)
    );
  }

  a.    jcc    (cc, jccStub);
  a.    jmp    (jccStub);
}

void CodeGenerator::cgAssertNonNull(IRInstruction* inst) {
  auto srcReg = m_regs[inst->src(0)].reg();
  auto dstReg = m_regs[inst->dst()].reg();
  if (RuntimeOption::EvalHHIRGenerateAsserts) {
    Label nonNull;
    m_as.testq (srcReg, srcReg);
    m_as.jne8  (nonNull);
    m_as.ud2();
    asm_label(m_as, nonNull);
  }
  emitMovRegReg(m_as, srcReg, dstReg);
}

void CodeGenerator::cgLdUnwinderValue(IRInstruction* inst) {
  cgLoad(rVmTl, TargetCache::kUnwinderTvOff, inst);
}

void CodeGenerator::cgBeginCatch(IRInstruction* inst) {
  auto const& info = m_state.catches[inst->block()];
  assert(info.afterCall);

  m_tx64->registerCatchTrace(info.afterCall, m_as.code.frontier);

  Stats::emitInc(m_as, Stats::TC_CatchTrace);

  // We want to restore state as though the call had completed
  // successfully, so skip over any stack arguments and pop any
  // saved registers.
  if (info.rspOffset) {
    m_as.addq(info.rspOffset, rsp);
  }
  PhysRegSaverParity::emitPops(m_as, info.savedRegs);
}

void CodeGenerator::cgEndCatch(IRInstruction* inst) {
  m_as.cmpb (0, rVmTl[TargetCache::kUnwinderSideExitOff]);
  unlikelyIfBlock(CC_E,
    [&](Asm& as) { // doSideExit == false, so call _Unwind_Resume
      as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi);
      as.call ((TCA)_Unwind_Resume); // pass control back to the unwinder
      as.ud2();
    });

  // doSideExit == true, so fall through to the side exit code
  Stats::emitInc(m_as, Stats::TC_CatchSideExit);
}

void CodeGenerator::cgDeleteUnwinderException(IRInstruction* inst) {
  m_as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi);
  m_as.call ((TCA)_Unwind_DeleteException);
}

void CodeGenerator::cgJcc(IRInstruction* inst) {
  emitCompare(inst->src(0), inst->src(1));
  emitFwdJcc(opToConditionCode(inst->op()), inst->taken());
}

void CodeGenerator::cgReqBindJcc(IRInstruction* inst) {
  // TODO(#2404427): prepareForTestAndSmash?
  emitCompare(inst->src(0), inst->src(1));
  emitReqBindJcc(opToConditionCode(inst->op()),
                 inst->extra<ReqBindJccData>());
}

#define X(x)                                                              \
  void CodeGenerator::cgReqBind##x(IRInstruction* i) { cgReqBindJcc(i); } \
  void CodeGenerator::cg##x       (IRInstruction* i) { cgJcc(i);        }

X(JmpGt);
X(JmpGte);
X(JmpLt);
X(JmpLte);
X(JmpEq);
X(JmpNeq);
X(JmpSame);
X(JmpNSame);

#undef X


/**
 * Once the arg sources and dests are all assigned; emit moves and exchanges to
 * put all the args in desired registers. Any arguments that don't fit in
 * registers will be put on the stack. In addition to moves and exchanges,
 * shuffleArgs also handles adding lea-offsets for dest registers (dest = src +
 * lea-offset) and zero extending bools (dest = zeroExtend(src)).
 */
typedef Transl::X64Assembler Asm;
static int64_t shuffleArgs(Asm& a, ArgGroup& args) {
  // Compute the move/shuffle plan.
  int moves[kNumRegs];
  ArgDesc* argDescs[kNumRegs];
  memset(moves, -1, sizeof moves);
  memset(argDescs, 0, sizeof argDescs);
  for (size_t i = 0; i < args.numRegArgs(); ++i) {
    auto kind = args[i].kind();
    if (!(kind == ArgDesc::Reg  ||
          kind == ArgDesc::Addr ||
          kind == ArgDesc::TypeReg)) {
      continue;
    }
    auto dstReg = args[i].dstReg();
    auto srcReg = args[i].srcReg();
    if (dstReg != srcReg) {
      moves[int(dstReg)] = int(srcReg);
      argDescs[int(dstReg)] = &args[i];
    }
  }
  std::vector<MoveInfo> howTo;
  doRegMoves(moves, int(rCgGP), howTo);

  // Execute the plan
  for (size_t i = 0; i < howTo.size(); ++i) {
    if (howTo[i].m_kind == MoveInfo::Move) {
      if (howTo[i].m_reg2 == rCgGP) {
        emitMovRegReg(a, howTo[i].m_reg1, howTo[i].m_reg2);
      } else {
        ArgDesc* argDesc = argDescs[int(howTo[i].m_reg2)];
        ArgDesc::Kind kind = argDesc->kind();
        if (kind == ArgDesc::Reg || kind == ArgDesc::TypeReg) {
          if (argDesc->isZeroExtend()) {
            assert(howTo[i].m_reg1.isGP());
            assert(howTo[i].m_reg2.isGP());
            a.    movzbl (rbyte(howTo[i].m_reg1), r32(howTo[i].m_reg2));
          } else {
            emitMovRegReg(a, howTo[i].m_reg1, howTo[i].m_reg2);
          }
        } else {
          assert(kind == ArgDesc::Addr);
          assert(howTo[i].m_reg1.isGP());
          assert(howTo[i].m_reg2.isGP());
          a.    lea    (howTo[i].m_reg1[argDesc->imm().q()],
                        howTo[i].m_reg2);
        }
        if (kind != ArgDesc::TypeReg) {
          argDesc->markDone();
        }
      }
    } else {
      assert(howTo[i].m_reg1.isGP());
      assert(howTo[i].m_reg2.isGP());
      a.    xchgq  (howTo[i].m_reg1, howTo[i].m_reg2);
    }
  }
  // Handle const-to-register moves, type shifting,
  // load-effective address and zero extending for bools.
  // Ignore args that have been handled by the
  // move above.
  for (size_t i = 0; i < args.numRegArgs(); ++i) {
    if (!args[i].done()) {
      ArgDesc::Kind kind = args[i].kind();
      PhysReg dst = args[i].dstReg();
      assert(dst.isGP());
      if (kind == ArgDesc::Imm) {
        a.emitImmReg(args[i].imm().q(), dst);
      } else if (kind == ArgDesc::TypeReg) {
        a.    shlq   (kTypeShiftBits, dst);
      } else if (kind == ArgDesc::Addr) {
        a.    addq   (args[i].imm(), dst);
      } else if (args[i].isZeroExtend()) {
        a.    movzbl (rbyte(dst), r32(dst));
      } else if (RuntimeOption::EvalHHIRGenerateAsserts &&
                 kind == ArgDesc::None) {
        a.emitImmReg(0xbadbadbadbadbad, dst);
      }
    }
  }

  // Store any remaining arguments to the stack
  for (int i = args.numStackArgs() - 1; i >= 0; --i) {
    auto& arg = args.stk(i);
    auto srcReg = arg.srcReg();
    assert(arg.dstReg() == InvalidReg);
    switch (arg.kind()) {
      case ArgDesc::Reg:
        if (arg.isZeroExtend()) {
          a.  movzbl(rbyte(srcReg), r32(rCgGP));
          a.  push(rCgGP);
        } else {
          if (srcReg.isXMM()) {
            emitMovRegReg(a, srcReg, rCgGP);
            a.push(rCgGP);
          } else {
            a.push(srcReg);
          }
        }
        break;

      case ArgDesc::TypeReg:
        static_assert(kTypeWordOffset == 4 || kTypeWordOffset == 1,
                      "kTypeWordOffset value not supported");
        assert(srcReg.isGP());
        // x86 stacks grow down, so push higher offset items first
        if (kTypeWordOffset == 4) {
          a.  pushl(r32(srcReg));
          // 4 bytes of garbage:
          a.  pushl(eax);
        } else {
          // 4 bytes of garbage:
          a.  pushl(eax);
          // get the type in the right place in rCgGP before pushing it
          a.  movb (rbyte(srcReg), rbyte(rCgGP));
          a.  shll (CHAR_BIT, r32(rCgGP));
          a.  pushl(r32(rCgGP));
        }
        break;

      case ArgDesc::Imm:
        a.    emitImmReg(arg.imm(), rCgGP);
        a.    push(rCgGP);
        break;

      case ArgDesc::Addr:
        not_implemented();

      case ArgDesc::None:
        a.    push(rax);
        if (RuntimeOption::EvalHHIRGenerateAsserts) {
          a.  storeq(0xbadbadbadbadbad, *rsp);
        }
        break;
    }
  }
  return args.numStackArgs() * sizeof(int64_t);
}

void CodeGenerator::cgCallNative(Asm& a, IRInstruction* inst) {
  using namespace NativeCalls;
  Opcode opc = inst->op();
  always_assert(CallMap::hasInfo(opc));

  const CallInfo& info = CallMap::info(opc);
  ArgGroup argGroup(m_regs);
  for (auto const& arg : info.args) {
    SSATmp* src = inst->src(arg.srcIdx);
    switch (arg.type) {
      case SSA:
        argGroup.ssa(src);
        break;
      case TV:
        argGroup.typedValue(src);
        break;
      case VecKeyS:
        argGroup.vectorKeyS(src);
        break;
      case VecKeyIS:
        argGroup.vectorKeyIS(src);
        break;
      case Immed:
        always_assert(0 && "We can't generate a native call for this");
        break;
    }
  }

  TCA addr = nullptr;
  switch (info.func.type) {
    case FPtr:
      addr = info.func.ptr;
      break;
    case FSSA:
      addr = inst->src(info.func.srcIdx)->getValTCA();
      break;
  }
  cgCallHelper(a,
               addr,
               info.dest != DestType::None ? inst->dst(0) : nullptr,
               info.sync,
               argGroup,
               info.dest);
}

void CodeGenerator::cgCallHelper(Asm& a,
                                 TCA addr,
                                 SSATmp* dst,
                                 SyncOptions sync,
                                 ArgGroup& args,
                                 DestType destType) {
  PhysReg dstReg0 = InvalidReg;
  PhysReg dstReg1 = InvalidReg;
  if (dst) {
    auto &info = m_regs[dst];
    dstReg0 = info.reg(0);
    dstReg1 = info.reg(1);
  }
  return cgCallHelper(a, Transl::CppCall(addr), dstReg0, dstReg1, sync, args,
                      destType);
}

void CodeGenerator::cgCallHelper(Asm& a,
                                 TCA addr,
                                 PhysReg dstReg,
                                 SyncOptions sync,
                                 ArgGroup& args,
                                 DestType destType) {
  cgCallHelper(a, Transl::CppCall(addr), dstReg, InvalidReg, sync, args,
               destType);
}

void CodeGenerator::cgCallHelper(Asm& a,
                                 const Transl::CppCall& call,
                                 PhysReg dstReg,
                                 SyncOptions sync,
                                 ArgGroup& args,
                                 DestType destType) {
  cgCallHelper(a, call, dstReg, InvalidReg, sync, args, destType);
}

void CodeGenerator::cgCallHelper(Asm& a,
                                 const Transl::CppCall& call,
                                 PhysReg dstReg0,
                                 PhysReg dstReg1,
                                 SyncOptions sync,
                                 ArgGroup& args,
                                 DestType destType) {
  cgCallHelper(a, call, dstReg0, dstReg1, sync, args,
               m_state.liveRegs[m_curInst], destType);
}

void CodeGenerator::cgCallHelper(Asm& a,
                                 const Transl::CppCall& call,
                                 PhysReg dstReg0,
                                 PhysReg dstReg1,
                                 SyncOptions sync,
                                 ArgGroup& args,
                                 RegSet toSave,
                                 DestType destType) {
  assert(m_curInst->isNative());

  // Save the caller-saved registers that are live across this
  // instruction. The number of regs to save and the number of args
  // being passed on the stack affect the parity of the PhysRegSaver,
  // so we use the generic version here.
  toSave = toSave & kCallerSaved;
  assert((toSave & RegSet().add(dstReg0).add(dstReg1)).empty());
  PhysRegSaverParity regSaver(1 + args.numStackArgs(), a, toSave);

  // Assign registers to the arguments then prepare them for the call.
  for (size_t i = 0; i < args.numRegArgs(); i++) {
    args[i].setDstReg(argNumToRegName[i]);
  }
  regSaver.bytesPushed(shuffleArgs(a, args));

  // do the call; may use a trampoline
  m_tx64->emitCall(a, call);
  if (sync != kNoSyncPoint) {
    recordSyncPoint(a, sync);
  }

  if (m_state.catchTrace) {
    auto& info = m_state.catches[m_state.catchTrace->front()];
    assert(!info.afterCall);
    info.afterCall = a.code.frontier;
    info.savedRegs = toSave;
    info.rspOffset = regSaver.rspAdjustment();
  }

  // copy the call result to the destination register(s)
  if (destType == DestType::TV) {
    // rax contains m_type and m_aux but we're expecting just the
    // type in the lower bits, so shift the type result register.
    auto rval = packed_tv ? reg::rdx : reg::rax;
    auto rtyp = packed_tv ? reg::rax : reg::rdx;
    if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, rtyp);
    shuffle2(a, rval, rtyp, dstReg0, dstReg1);
  } else if (destType == DestType::SSA) {
    // copy the single-register result to dstReg0
    assert(dstReg1 == InvalidReg);
    if (dstReg0 != InvalidReg) emitMovRegReg(a, reg::rax, dstReg0);
  } else {
    // void return type, no registers have values
    assert(dstReg0 == InvalidReg && dstReg1 == InvalidReg);
  }
}

/*
 * This doesn't really produce any code; it just keeps track of the current
 * bytecode offset.
 */
void CodeGenerator::cgMarker(IRInstruction* inst) {
  m_curBcOff = inst->extra<MarkerData>()->bcOff;
}

void CodeGenerator::cgMov(IRInstruction* inst) {
  assert(!m_regs[inst->src(0)].hasReg(1));//TODO: t2082361: handle Gen & Cell
  SSATmp* dst   = inst->dst();
  SSATmp* src   = inst->src(0);
  auto dstReg = m_regs[dst].reg();
  if (!m_regs[src].hasReg(0)) {
    assert(src->isConst());
    if (src->type() == Type::Bool) {
      emitLoadImm(m_as, (int64_t)src->getValBool(), dstReg);
    } else {
      emitLoadImm(m_as, src->getValRawInt(), dstReg);
    }
  } else {
    auto srcReg = m_regs[src].reg();
    emitMovRegReg(m_as, srcReg, dstReg);
  }
}

template<class OpInstr, class Oper>
void CodeGenerator::cgUnaryIntOp(SSATmp* dst,
                                 SSATmp* src,
                                 OpInstr instr,
                                 Oper oper) {
  if (src->type() != Type::Int && src->type() != Type::Bool) {
    assert(0); CG_PUNT(UnaryIntOp);
  }
  auto dstReg = m_regs[dst].reg();
  auto srcReg = m_regs[src].reg();
  assert(dstReg != InvalidReg);
  auto& a = m_as;

  // Integer operations require 64-bit representations
  zeroExtendIfBool(a, src, m_regs[src]);

  if (srcReg != InvalidReg) {
    emitMovRegReg(a, srcReg, dstReg);
    (a.*instr)   (dstReg);
  } else {
    assert(src->isConst());
    emitLoadImm(a, oper(src->getValRawInt()), dstReg);
  }
}

void CodeGenerator::cgNegateWork(SSATmp* dst, SSATmp* src) {
  cgUnaryIntOp(dst, src, &Asm::neg, [](int64_t i) { return -i; });
}

inline static Reg8 convertToReg8(PhysReg reg) { return rbyte(reg); }
inline static Reg64 convertToReg64(PhysReg reg) { return reg; }

template<class Oper, class RegType>
void CodeGenerator::cgBinaryIntOp(IRInstruction* inst,
              void (Asm::*instrIR)(Immed, RegType),
              void (Asm::*instrRR)(RegType, RegType),
              void (Asm::*movInstr)(RegType, RegType),
              Oper oper,
              RegType (*convertReg)(PhysReg),
              Commutativity commuteFlag) {
  const SSATmp* dst   = inst->dst();
  const SSATmp* src1  = inst->src(0);
  const SSATmp* src2  = inst->src(1);
  if (!(src1->isA(Type::Bool) || src1->isA(Type::Int)) ||
      !(src2->isA(Type::Bool) || src2->isA(Type::Int))) {
    CG_PUNT(cgBinaryIntOp);
  }

  bool const commutative = commuteFlag == Commutative;
  auto const dstReg      = m_regs[dst].reg();
  auto const src1Reg     = m_regs[src1].reg();
  auto const src2Reg     = m_regs[src2].reg();
  auto& a                = m_as;

  auto const dstOpReg    = convertReg(dstReg);
  auto const src1OpReg   = convertReg(src1Reg);
  auto const src2OpReg   = convertReg(src2Reg);
  auto const rOpScratch  = convertReg(m_rScratch);

  // Two registers.
  if (src1Reg != InvalidReg && src2Reg != InvalidReg) {
    if (dstReg == src1Reg) {
      (a.*instrRR)  (src2OpReg, dstOpReg);
    } else if (dstReg == src2Reg) {
      if (commutative) {
        (a.*instrRR) (src1OpReg, dstOpReg);
      } else {
        (a.*movInstr)(src1OpReg, rOpScratch);
        (a.*instrRR) (src2OpReg, rOpScratch);
        (a.*movInstr)(rOpScratch, dstOpReg);
      }
    } else {
      emitMovRegReg(a, src1Reg, dstReg);
      (a.*instrRR) (src2OpReg, dstOpReg);
    }
    return;
  }

  // Two immediates.
  if (src1Reg == InvalidReg && src2Reg == InvalidReg) {
    assert(src1->isConst() && src2->isConst());
    int64_t value = oper(src1->getValRawInt(), src2->getValRawInt());
    emitLoadImm(a, value, dstReg);
    return;
  }

  // One register, and one immediate.
  if (commutative) {
    auto immedSrc = (src2Reg == InvalidReg ? src2 : src1);
    auto immed = immedSrc->getValRawInt();
    auto srcReg = m_regs[(src2Reg == InvalidReg ? src1 : src2)].reg();
    if (srcReg == dstReg) {
      (a.*instrIR) (immed, dstOpReg);
    } else {
      emitLoadImm(a, immed, dstReg);
      (a.*instrRR) (convertReg(srcReg), dstOpReg);
    }
    return;
  }

  // NonCommutative:
  if (src1Reg == InvalidReg) {
    if (dstReg == src2Reg) {
      emitLoadImm(a, src1->getValRawInt(), m_rScratch);
      (a.*instrRR) (src2OpReg, rOpScratch);
      (a.*movInstr)(rOpScratch, dstOpReg);
    } else {
      emitLoadImm(a, src1->getValRawInt(), dstReg);
      (a.*instrRR) (src2OpReg, dstOpReg);
    }
    return;
  }

  assert(src2Reg == InvalidReg);
  emitMovRegReg(a, src1Reg, dstReg);
  (a.*instrIR) (src2->getValRawInt(), dstOpReg);
}

template<class Oper, class RegType>
void CodeGenerator::cgBinaryOp(IRInstruction* inst,
                 void (Asm::*instrIR)(Immed, RegType),
                 void (Asm::*instrRR)(RegType, RegType),
                 void (Asm::*movInstr)(RegType, RegType),
                 void (Asm::*fpInstr)(RegXMM, RegXMM),
                 Oper oper,
                 RegType (*convertReg)(PhysReg),
                 Commutativity commuteFlag) {
  const SSATmp* dst   = inst->dst();
  const SSATmp* src1  = inst->src(0);
  const SSATmp* src2  = inst->src(1);
  if (!(src1->isA(Type::Bool) || src1->isA(Type::Int) || src1->isA(Type::Dbl))
      ||
      !(src2->isA(Type::Bool) || src2->isA(Type::Int) || src2->isA(Type::Dbl)) )
  {
    CG_PUNT(cgBinaryOp);
  }
  if (src1->isA(Type::Dbl) || src2->isA(Type::Dbl)) {
    PhysReg dstReg  = m_regs[dst].reg();
    PhysReg resReg  = dstReg.isXMM() && dstReg != m_regs[src2].reg() ?
                      dstReg : PhysReg(rCgXMM0);
    assert(resReg.isXMM());

    PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, resReg);
    PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
    assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);

    emitMovRegReg(m_as, srcReg1, resReg);

    (m_as.*fpInstr)(srcReg2, resReg);

    emitMovRegReg(m_as, resReg, dstReg);
    return;
  }
  cgBinaryIntOp(inst, instrIR, instrRR, movInstr,
                oper, convertReg, commuteFlag);
}

bool CodeGenerator::emitIncDecHelper(SSATmp* dst, SSATmp* src1, SSATmp* src2,
                                     void(Asm::*emitFunc)(Reg64)) {
  if (m_regs[src1].reg() != InvalidReg &&
      m_regs[dst].reg() != InvalidReg &&
      src1->isA(Type::Int) &&
      // src2 == 1:
      src2->isConst() && src2->isA(Type::Int) && src2->getValInt() == 1) {
    emitMovRegReg(m_as, m_regs[src1].reg(), m_regs[dst].reg());
    (m_as.*emitFunc)(m_regs[dst].reg());
    return true;
  }
  return false;
}

/*
 * If src2 is 1, this generates dst = src1 + 1 using the "inc" x86 instruction.
 * The return value is whether or not the instruction could be generated.
 */
bool CodeGenerator::emitInc(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
  return emitIncDecHelper(dst, src1, src2, &Asm::incq);
}

/*
 * If src2 is 1, this generates dst = src1 - 1 using the "dec" x86 instruction.
 * The return value is whether or not the instruction could be generated.
 */
bool CodeGenerator::emitDec(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
  return emitIncDecHelper(dst, src1, src2, &Asm::decq);
}

void CodeGenerator::cgOpAdd(IRInstruction* inst) {
  SSATmp* dst  = inst->dst();
  SSATmp* src1 = inst->src(0);
  SSATmp* src2 = inst->src(1);

  // Special cases: x = y + 1
  if (emitInc(dst, src1, src2) || emitInc(dst, src2, src1)) return;

  cgBinaryOp(inst,
             &Asm::addq,
             &Asm::addq,
             &Asm::movq,
             &Asm::addsd_xmm_xmm,
             std::plus<int64_t>(),
             &convertToReg64,
             Commutative);
}

void CodeGenerator::cgOpSub(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* src1  = inst->src(0);
  SSATmp* src2  = inst->src(1);

  if (emitDec(dst, src1, src2)) return;

  if (src1->isConst() && src1->isA(Type::Int) && src1->getValInt() == 0 &&
      src2->isA(Type::Int)) {
    cgNegateWork(dst, src2);
    return;
  }

  cgBinaryOp(inst,
             &Asm::subq,
             &Asm::subq,
             &Asm::movq,
             &Asm::subsd_xmm_xmm,
             std::minus<int64_t>(),
             &convertToReg64,
             NonCommutative);
}

void CodeGenerator::cgOpDiv(IRInstruction* inst) {
  not_implemented();
}

void CodeGenerator::cgOpBitAnd(IRInstruction* inst) {
  cgBinaryIntOp(inst,
                &Asm::andq,
                &Asm::andq,
                &Asm::movq,
                [] (int64_t a, int64_t b) { return a & b; },
                &convertToReg64,
                Commutative);
}

void CodeGenerator::cgOpBitOr(IRInstruction* inst) {
  cgBinaryIntOp(inst,
                &Asm::orq,
                &Asm::orq,
                &Asm::movq,
                [] (int64_t a, int64_t b) { return a | b; },
                &convertToReg64,
                Commutative);
}

void CodeGenerator::cgOpBitXor(IRInstruction* inst) {
  cgBinaryIntOp(inst,
                &Asm::xorq,
                &Asm::xorq,
                &Asm::movq,
                [] (int64_t a, int64_t b) { return a ^ b; },
                &convertToReg64,
                Commutative);
}

void CodeGenerator::cgOpBitNot(IRInstruction* inst) {
  cgUnaryIntOp(inst->dst(),
               inst->src(0),
               &Asm::not,
               [](int64_t i) { return ~i; });
}

void CodeGenerator::cgOpLogicXor(IRInstruction* inst) {
  cgBinaryIntOp(inst,
                &Asm::xorb,
                &Asm::xorb,
                &Asm::movb,
                [] (bool a, bool b) { return a ^ b; },
                &convertToReg8,
                Commutative);
}

void CodeGenerator::cgOpMul(IRInstruction* inst) {
  cgBinaryOp(inst,
             &Asm::imul,
             &Asm::imul,
             &Asm::movq,
             &Asm::mulsd_xmm_xmm,
             std::multiplies<int64_t>(),
             &convertToReg64,
             Commutative);
}

void CodeGenerator::cgOpNot(IRInstruction* inst) {
  auto const src = inst->src(0);
  auto const dstReg = m_regs[inst->dst()].reg();
  auto& a = m_as;

  if (src->isConst()) {
    a.    movb   (!src->getValBool(), rbyte(dstReg));
  } else {
    if (dstReg != m_regs[src].reg()) {
      a.  movb   (rbyte(m_regs[src].reg()), rbyte(dstReg));
    }
    a.    xorb   (1, rbyte(dstReg));
  }
}

///////////////////////////////////////////////////////////////////////////////
// Comparison Operators
///////////////////////////////////////////////////////////////////////////////

#define DISPATCHER(name)                                                \
  HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, StringData* a2)      \
  { return name(a1, a2); }                                              \
  HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, int64_t a2)            \
  { return name(a1, a2); }                                              \
  HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, ObjectData* a2)      \
  { return name(a1, Object(a2)); }                                      \
  HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, ObjectData* a2)      \
  { return name(Object(a1), Object(a2)); }                              \
  HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, int64_t a2)            \
  { return name(Object(a1), a2); }                                      \
  HOT_FUNC_VM int64_t ccmp_ ## name (ArrayData* a1, ArrayData* a2)        \
  { return name(Array(a1), Array(a2)); }

DISPATCHER(same)
DISPATCHER(equal)
DISPATCHER(more)
DISPATCHER(less)

#undef DISPATCHER

template <typename A, typename B>
inline int64_t ccmp_nsame(A a, B b) { return !ccmp_same(a, b); }

template <typename A, typename B>
inline int64_t ccmp_nequal(A a, B b) { return !ccmp_equal(a, b); }

template <typename A, typename B>
inline int64_t ccmp_lte(A a, B b) { return !ccmp_more(a, b); }

template <typename A, typename B>
inline int64_t ccmp_gte(A a, B b) { return !ccmp_less(a, b); }

#define CG_OP_CMP(inst, setter, name)                                         \
  cgOpCmpHelper(inst, &Asm:: setter, ccmp_ ## name, ccmp_ ## name,            \
                ccmp_ ## name, ccmp_ ## name, ccmp_ ## name, ccmp_ ## name)

// SRON - string, resource, object, or number
static bool typeIsSRON(Type t) {
  return t.isString()
      || t == Type::Obj // encompases object and resource
      || t == Type::Int
      || t == Type::Dbl
      ;
}

void CodeGenerator::cgOpCmpHelper(
          IRInstruction* inst,
          void (Asm::*setter)(Reg8),
          int64_t (*str_cmp_str)(StringData*, StringData*),
          int64_t (*str_cmp_int)(StringData*, int64_t),
          int64_t (*str_cmp_obj)(StringData*, ObjectData*),
          int64_t (*obj_cmp_obj)(ObjectData*, ObjectData*),
          int64_t (*obj_cmp_int)(ObjectData*, int64_t),
          int64_t (*arr_cmp_arr)( ArrayData*, ArrayData*)
        ) {
  SSATmp* dst   = inst->dst();
  SSATmp* src1  = inst->src(0);
  SSATmp* src2  = inst->src(1);

  Type type1 = src1->type();
  Type type2 = src2->type();

  auto src1Reg = m_regs[src1].reg();
  auto src2Reg = m_regs[src2].reg();
  auto dstReg  = m_regs[dst].reg();

  auto setFromFlags = [&] {
    (m_as.*setter)(rbyte(dstReg));
  };
  // It is possible that some pass has been done after simplification; if such
  // a pass invalidates our invariants, then just punt.

  // simplifyCmp has done const-const optimization
  //
  // If the types are the same and there is only one constant,
  // simplifyCmp has moved it to the right.
  if (src1->isConst()) {
    CG_PUNT(cgOpCmpHelper_const);
  }

  /////////////////////////////////////////////////////////////////////////////
  // case 1: null/string cmp string
  // simplifyCmp has converted the null to ""
  if (type1.isString() && type2.isString()) {
    ArgGroup args(m_regs);
    args.ssa(src1).ssa(src2);
    cgCallHelper(m_as, (TCA)str_cmp_str,  dst, kSyncPoint, args);
  }

  /////////////////////////////////////////////////////////////////////////////
  // case 2: bool/null cmp anything
  // simplifyCmp has converted all args to bool
  else if (type1 == Type::Bool && type2 == Type::Bool) {
    if (src2->isConst()) {
      m_as.    cmpb (src2->getValBool(), Reg8(int(src1Reg)));
    } else {
      m_as.    cmpb (Reg8(int(src2Reg)), Reg8(int(src1Reg)));
    }
    setFromFlags();
  }

  /////////////////////////////////////////////////////////////////////////////
  // case 3, 4, and 7: string/resource/object/number (sron) cmp sron
  // These cases must be amalgamated because Type::Obj can refer to an object
  //  or to a resource.
  // strings are canonicalized to the left, ints to the right
  else if (typeIsSRON(type1) && typeIsSRON(type2)) {
    // the common case: int cmp int
    if (type1 == Type::Int && type2 == Type::Int) {
      if (src2->isConst()) {
        m_as.cmp_imm64_reg64(src2->getValInt(), src1Reg);
      } else {
        m_as.cmp_reg64_reg64(src2Reg, src1Reg);
      }
      setFromFlags();
    }

    else if (type1 == Type::Dbl || type2 == Type::Dbl) {
      if ((type1 == Type::Dbl || type1 == Type::Int) &&
          (type2 == Type::Dbl || type2 == Type::Int)) {
        PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, rCgXMM0);
        PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
        assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);
        doubleCmp(m_as, srcReg1, srcReg2);
        setFromFlags();
      } else {
        CG_PUNT(cgOpCmpHelper_Dbl);
      }
    }

    else if (type1.isString()) {
      // string cmp string is dealt with in case 1
      // string cmp double is punted above

      if (type2 == Type::Int) {
        ArgGroup args(m_regs);
        args.ssa(src1).ssa(src2);
        cgCallHelper(m_as, (TCA)str_cmp_int,  dst, kSyncPoint, args);
      } else if (type2 == Type::Obj) {
        ArgGroup args(m_regs);
        args.ssa(src1).ssa(src2);
        cgCallHelper(m_as, (TCA)str_cmp_obj,  dst, kSyncPoint, args);
      } else {
        CG_PUNT(cgOpCmpHelper_sx);
      }
    }

    else if (type1 == Type::Obj) {
      // string cmp object/resource is dealt with above
      // object cmp double is punted above

      if (type2 == Type::Obj) {
        ArgGroup args(m_regs);
        args.ssa(src1).ssa(src2);
        cgCallHelper(m_as, (TCA)obj_cmp_obj,  dst, kSyncPoint, args);
      } else if (type2 == Type::Int) {
        ArgGroup args(m_regs);
        args.ssa(src1).ssa(src2);
        cgCallHelper(m_as, (TCA)obj_cmp_int,  dst, kSyncPoint, args);
      } else {
        CG_PUNT(cgOpCmpHelper_ox);
      }
    }

   else NOT_REACHED();
  }

  /////////////////////////////////////////////////////////////////////////////
  // case 5: array cmp array
  else if (type1.isArray() && type2.isArray()) {
    ArgGroup args(m_regs);
    args.ssa(src1).ssa(src2);
    cgCallHelper(m_as, (TCA)arr_cmp_arr,  dst, kSyncPoint, args);
  }

  /////////////////////////////////////////////////////////////////////////////
  // case 6: array cmp anything
  // simplifyCmp has already dealt with this case.

  /////////////////////////////////////////////////////////////////////////////
  else {
    // We have a type which is not a common type. It might be a cell or a box.
    CG_PUNT(cgOpCmpHelper_unimplemented);
  }
}

void CodeGenerator::cgOpEq(IRInstruction* inst) {
  CG_OP_CMP(inst, sete, equal);
}

void CodeGenerator::cgOpNeq(IRInstruction* inst) {
  CG_OP_CMP(inst, setne, nequal);
}

void CodeGenerator::cgOpSame(IRInstruction* inst) {
  CG_OP_CMP(inst, sete, same);
}

void CodeGenerator::cgOpNSame(IRInstruction* inst) {
  CG_OP_CMP(inst, setne, nsame);
}

void CodeGenerator::cgOpLt(IRInstruction* inst) {
  CG_OP_CMP(inst, setl, less);
}

void CodeGenerator::cgOpGt(IRInstruction* inst) {
  CG_OP_CMP(inst, setg, more);
}

void CodeGenerator::cgOpLte(IRInstruction* inst) {
  CG_OP_CMP(inst, setle, lte);
}

void CodeGenerator::cgOpGte(IRInstruction* inst) {
  CG_OP_CMP(inst, setge, gte);
}

///////////////////////////////////////////////////////////////////////////////
// Type check operators
///////////////////////////////////////////////////////////////////////////////

// Overloads to put the ObjectData* into a register so emitTypeTest
// can cmp to the Class* expected by the specialized Type

// Nothing to do, return the register that contain the ObjectData already
Reg64 getObjectDataEnregistered(Asm& as, PhysReg dataSrc, Reg64 scratch) {
  return dataSrc;
}

// Enregister the meoryRef so it can be used with an offset by the
// cmp instruction
Reg64 getObjectDataEnregistered(Asm& as,
                                MemoryRef dataSrc,
                                Reg64 scratch) {
  as.loadq(dataSrc, scratch);
  return scratch;
}

template<class Loc1, class Loc2, class JmpFn>
void CodeGenerator::emitTypeTest(Type type, Loc1 typeSrc, Loc2 dataSrc,
                                 JmpFn doJcc) {
  assert(!type.subtypeOf(Type::Cls));
  ConditionCode cc;
  if (type.isString()) {
    emitTestTVType(m_as, KindOfStringBit, typeSrc);
    cc = CC_NZ;
  } else if (type.equals(Type::UncountedInit)) {
    emitTestTVType(m_as, KindOfUncountedInitBit, typeSrc);
    cc = CC_NZ;
  } else if (type.equals(Type::Uncounted)) {
    emitCmpTVType(m_as, KindOfRefCountThreshold, typeSrc);
    cc = CC_LE;
  } else if (type.equals(Type::Cell)) {
    emitCmpTVType(m_as, KindOfRef, typeSrc);
    cc = CC_L;
  } else if (type.equals(Type::Gen)) {
    // nothing to check
    return;
  } else {
    DataType dataType = type.toDataType();
    assert(dataType == KindOfRef ||
           (dataType >= KindOfUninit && dataType <= KindOfObject));
    emitCmpTVType(m_as, dataType, typeSrc);
    cc = CC_E;
  }
  doJcc(cc);
  if (type.strictSubtypeOf(Type::Obj)) {
    // emit the specific class test
    assert(type.getClass()->attrs() & AttrFinal);
    auto reg = getObjectDataEnregistered(m_as, dataSrc, m_rScratch);
    m_as.cmpq(type.getClass(), reg[ObjectData::getVMClassOffset()]);
    doJcc(cc);
  }
}

template<class JmpFn>
void CodeGenerator::emitIsTypeTest(IRInstruction* inst, JmpFn doJcc) {
  auto const src = inst->src(0);

  // punt if specialized object for now
  if (inst->typeParam().strictSubtypeOf(Type::Obj)) {
    CG_PUNT(IsType-SpecializedUnsupported);
  }
  if (inst->typeParam().equals(Type::Obj)) {
    auto const srcReg = m_regs[src].reg();
    if (src->isA(Type::PtrToGen)) {
      emitTestTVType(m_as, KindOfObject, srcReg[TVOFF(m_type)]);
      TCA toPatch = m_as.code.frontier;
      m_as.   jne8(toPatch);  // 1

      // Get the ObjectData*
      emitDeref(m_as, srcReg, m_rScratch);
      m_as.   cmpq(SystemLib::s_resourceClass,
                   m_rScratch[ObjectData::getVMClassOffset()]);
      // 1:
      m_as.patchJcc8(toPatch, m_as.code.frontier);
    } else {
      // Cases where src isn't an Obj should have been simplified away
      if (!src->isA(Type::Obj)) {
        CG_PUNT(IsType-KnownWrongType);
      }
      m_as.   cmpq(SystemLib::s_resourceClass,
                   srcReg[ObjectData::getVMClassOffset()]);
    }
    // At this point, the flags say "equal" if is_object is false.
    doJcc(CC_NE);
    return;
  }

  if (src->isA(Type::PtrToGen)) {
    PhysReg base = m_regs[src].reg();
    emitTypeTest(inst->typeParam(), base[TVOFF(m_type)],
                 base[TVOFF(m_data)],
      [&](ConditionCode cc) { doJcc(cc); });
    return;
  }
  assert(src->isA(Type::Gen));
  assert(!src->isConst());

  PhysReg typeSrcReg = m_regs[src].reg(1); // type register
  if (typeSrcReg == InvalidReg) {
    CG_PUNT(IsType-KnownType);
  }
  PhysReg dataSrcReg = m_regs[src].reg(); // data register
  emitTypeTest(inst->typeParam(), typeSrcReg, dataSrcReg,
    [&](ConditionCode cc) { doJcc(cc); });
}

template<class Loc>
void CodeGenerator::emitTypeCheck(Type type,
                                  Loc typeSrc,
                                  Loc dataSrc,
                                  Block* taken) {
  emitTypeTest(type, typeSrc, dataSrc,
    [&](ConditionCode cc) {
      emitFwdJcc(ccNegate(cc), taken);
    });
}

template<class Loc>
void CodeGenerator::emitTypeGuard(Type type, Loc typeSrc, Loc dataSrc) {
  emitTypeTest(type, typeSrc, dataSrc,
    [&](ConditionCode cc) {
      auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
      auto const destSR = m_tx64->getSrcRec(destSK);
      m_tx64->emitFallbackCondJmp(m_as, *destSR, ccNegate(cc));
    });
}

void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) {
  m_as.setcc(cc, rbyte(m_regs[inst->dst()].reg()));
}

void CodeGenerator::cgIsTypeMemCommon(IRInstruction* inst, bool negate) {
  bool called = false; // check emitSetCc is called only once
  emitIsTypeTest(inst,
    [&](ConditionCode cc) {
      assert(!called);
      emitSetCc(inst, negate ? ccNegate(cc) : cc);
      called = true;
    });
}

void CodeGenerator::cgIsTypeCommon(IRInstruction* inst, bool negate) {
  bool called = false; // check emitSetCc is called only once
  emitIsTypeTest(inst,
    [&](ConditionCode cc) {
      assert(!called);
      emitSetCc(inst, negate ? ccNegate(cc) : cc);
      called = true;
    });
}

void CodeGenerator::cgJmpIsTypeCommon(IRInstruction* inst, bool negate) {
  emitIsTypeTest(inst,
    [&](ConditionCode cc) {
      emitFwdJcc(negate ? ccNegate(cc) : cc,  inst->taken());
    });
}

void CodeGenerator::cgIsType(IRInstruction* inst) {
  cgIsTypeCommon(inst, false);
}

void CodeGenerator::cgIsNType(IRInstruction* inst) {
  cgIsTypeCommon(inst, true);
}

// TODO(#2404341): remove JmpIs{N,}Type

void CodeGenerator::cgJmpIsType(IRInstruction* inst) {
  cgJmpIsTypeCommon(inst, false);
}

void CodeGenerator::cgJmpIsNType(IRInstruction* inst) {
  cgJmpIsTypeCommon(inst, true);
}

void CodeGenerator::cgIsTypeMem(IRInstruction* inst) {
  cgIsTypeMemCommon(inst, false);
}

void CodeGenerator::cgIsNTypeMem(IRInstruction* inst) {
  cgIsTypeMemCommon(inst, true);
}

///////////////////////////////////////////////////////////////////////////////

HOT_FUNC_VM static bool instanceOfHelper(const Class* objClass,
                                         const Class* testClass) {
  return testClass && objClass->classof(testClass);
}

void CodeGenerator::cgInstanceOf(IRInstruction* inst) {
  cgCallHelper(m_as,
               TCA(instanceOfHelper),
               inst->dst(),
               kNoSyncPoint,
               ArgGroup(m_regs)
                 .ssa(inst->src(0))
                 .ssa(inst->src(1)));
}

/*
 * Check instanceof using instance bitmasks.
 *
 * Note it's not necessary to check whether the test class is defined:
 * if it doesn't exist than the candidate can't be an instance of it
 * and will fail this check.
 */
void CodeGenerator::emitInstanceBitmaskCheck(IRInstruction* inst) {
  auto const rObjClass     = m_regs[inst->src(0)].reg(0);
  auto const testClassName = inst->src(1)->getValStr();
  auto& a = m_as;

  int offset;
  uint8_t mask;
  if (!Class::getInstanceBitMask(testClassName, offset, mask)) {
    always_assert(!"cgInstanceOfBitmask had no bitmask");
  }
  a.    testb  (int8_t(mask), rObjClass[offset]);
}

void CodeGenerator::cgInstanceOfBitmask(IRInstruction* inst) {
  auto& a = m_as;
  emitInstanceBitmaskCheck(inst);
  a.    setnz  (rbyte(m_regs[inst->dst()].reg()));
}

void CodeGenerator::cgNInstanceOfBitmask(IRInstruction* inst) {
  auto& a = m_as;
  emitInstanceBitmaskCheck(inst);
  a.    setz   (rbyte(m_regs[inst->dst()].reg()));
}

void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction* inst) {
  emitInstanceBitmaskCheck(inst);
  emitFwdJcc(CC_NZ, inst->taken());
}

void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction* inst) {
  emitInstanceBitmaskCheck(inst);
  emitFwdJcc(CC_Z, inst->taken());
}

void CodeGenerator::cgReqBindJmpInstanceOfBitmask(IRInstruction* inst) {
  emitInstanceBitmaskCheck(inst);
  emitReqBindJcc(opToConditionCode(inst->op()),
                 inst->extra<ReqBindJccData>());
}

void CodeGenerator::cgReqBindJmpNInstanceOfBitmask(IRInstruction* inst) {
  emitInstanceBitmaskCheck(inst);
  emitReqBindJcc(opToConditionCode(inst->op()),
                 inst->extra<ReqBindJccData>());
}

/*
 * Check instanceof using the superclass vector on the end of the
 * Class entry.
 */
void CodeGenerator::cgExtendsClass(IRInstruction* inst) {
  auto const rObjClass     = m_regs[inst->src(0)].reg();
  auto const testClass     = inst->src(1)->getValClass();
  auto rTestClass          = m_regs[inst->src(1)].reg();
  auto const rdst          = rbyte(m_regs[inst->dst()].reg());
  auto& a = m_as;

  Label out;
  Label notExact;
  Label falseLabel;

  if (rTestClass == InvalidReg) { // TODO(#2031606)
    rTestClass = m_rScratch; // careful below about asm-x64 smashing this
    emitLoadImm(a, (int64_t)testClass, rTestClass);
  }

  // Test if it is the exact same class.  TODO(#2044801): we should be
  // doing this control flow at the IR level.
  if (!(testClass->attrs() & AttrAbstract)) {
    a.    cmpq   (rTestClass, rObjClass);
    a.    jne8   (notExact);
    a.    movb   (1, rdst);
    a.    jmp8   (out);
  }

  auto const vecOffset = Class::classVecOff() +
    sizeof(Class*) * (testClass->classVecLen() - 1);

  // Check the length of the class vectors---if the candidate's is at
  // least as long as the potential base (testClass) it might be a
  // subclass.
asm_label(a, notExact);
  a.    cmpl   (testClass->classVecLen(),
                rObjClass[Class::classVecLenOff()]);
  a.    jb8    (falseLabel);

  // If it's a subclass, rTestClass must be at the appropriate index.
  a.    cmpq   (rTestClass, rObjClass[vecOffset]);
  a.    sete   (rdst);
  a.    jmp8   (out);

asm_label(a, falseLabel);
  a.    xorl   (r32(rdst), r32(rdst));

asm_label(a, out);
}

void CodeGenerator::cgConvDblToBool(IRInstruction* inst) {
  SSATmp* dst = inst->dst();
  auto dstReg = m_regs[dst].reg();
  assert(dstReg != InvalidReg);
  SSATmp* src = inst->src(0);
  auto srcReg = m_regs[src].reg();
  if (srcReg == InvalidReg) {
    assert(src->isConst());
    double constVal = src->getValDbl();
    if (constVal == 0.0) {
      m_as.xor_reg64_reg64(dstReg, dstReg);
    } else {
      m_as.mov_imm64_reg(1, dstReg);
    }
  } else {
    emitMovRegReg(m_as, srcReg, dstReg);
    m_as.shlq(1, dstReg); // 0.0 stays zero and -0.0 is now 0.0
    m_as.setne(rbyte(dstReg)); // lower byte becomes 1 if dstReg != 0
    m_as.movzbl(rbyte(dstReg), r32(dstReg));
  }
}

void CodeGenerator::cgConvIntToBool(IRInstruction* inst) {
  SSATmp* dst = inst->dst();
  auto dstReg = m_regs[dst].reg();
  assert(dstReg != InvalidReg);
  SSATmp* src = inst->src(0);
  auto srcReg = m_regs[src].reg();

  if (srcReg == InvalidReg) {
    assert(src->isConst());
    int64_t constVal = src->getValInt();
    if (constVal == 0) {
      m_as.xor_reg64_reg64(dstReg, dstReg);
    } else {
      m_as.mov_imm64_reg(1, dstReg);
    }
  } else {
    m_as.test_reg64_reg64(srcReg, srcReg);
    m_as.setne(rbyte(dstReg));
    m_as.movzbl(rbyte(dstReg), r32(dstReg));
  }
}

void CodeGenerator::emitConvBoolOrIntToDbl(IRInstruction* inst) {
  SSATmp* src = inst->src(0);
  SSATmp* dst = inst->dst();
  PhysReg dstReg = m_regs[dst].reg();
  assert(src->isA(Type::Bool) || src->isA(Type::Int));
  assert(dstReg != InvalidReg);
  if (src->isConst()) {
    int64_t constVal = src->getValRawInt();
    if (src->isA(Type::Bool)) constVal = constVal != 0; // see task #2401790
    constVal = convIntToDouble(constVal);
    emitLoadImm(m_as, constVal, dstReg);
  } else {
    // cvtsi2sd doesn't modify the high bits of its target, which can
    // cause false dependencies to prevent register renaming from kicking
    // in. Break the dependency chain by zeroing out the XMM reg.
    PhysReg srcReg = m_regs[src].reg();
    PhysReg xmmReg = dstReg.isXMM() ? dstReg : PhysReg(rCgXMM0);
    m_as.pxor_xmm_xmm(xmmReg, xmmReg);
    m_as.cvtsi2sd_reg64_xmm(srcReg, xmmReg);
    zeroExtendIfBool(m_as, src, m_regs[src]);
    emitMovRegReg(m_as, xmmReg, dstReg);
  }
}

void CodeGenerator::cgConvBoolToDbl(IRInstruction* inst) {
  emitConvBoolOrIntToDbl(inst);
}

void CodeGenerator::cgConvIntToDbl(IRInstruction* inst) {
  emitConvBoolOrIntToDbl(inst);
}

void CodeGenerator::cgConvBoolToInt(IRInstruction* inst) {
  SSATmp* dst = inst->dst();
  auto dstReg = m_regs[dst].reg();
  assert(dstReg != InvalidReg);
  SSATmp* src = inst->src(0);
  auto srcReg = m_regs[src].reg();
  assert(src->isConst() == (srcReg == InvalidReg));
  if (srcReg == InvalidReg) {
    int64_t constVal = src->getValRawInt();
    if (constVal == 0) {
      m_as.xor_reg64_reg64(dstReg, dstReg);
    } else {
      m_as.mov_imm64_reg(1, dstReg);
    }
  } else {
    m_as.movzbl(rbyte(srcReg), r32(dstReg));
  }
}

void CodeGenerator::cgConvBoolToStr(IRInstruction* inst) {
  SSATmp* dst = inst->dst();
  auto dstReg = m_regs[dst].reg();
  assert(dstReg != InvalidReg);
  SSATmp* src = inst->src(0);
  auto srcReg = m_regs[src].reg();
  assert(src->isConst() == (srcReg == InvalidReg));
  if (srcReg == InvalidReg) {
    auto constVal = src->getValBool();
    if (!constVal) {
      m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
    } else {
      m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), dstReg);
    }
  } else {
    m_as.testb(Reg8(int(srcReg)), Reg8(int(srcReg)));
    m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
    m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), m_rScratch);
    m_as.cmov_reg64_reg64(CC_NZ, m_rScratch, dstReg);
  }
}

void CodeGenerator::cgUnboxPtr(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* src   = inst->src(0);

  auto srcReg = m_regs[src].reg();
  auto dstReg = m_regs[dst].reg();

  assert(srcReg != InvalidReg);
  assert(dstReg != InvalidReg);

  emitMovRegReg(m_as, srcReg, dstReg);
  emitDerefIfVariant(m_as, PhysReg(dstReg));
}

void CodeGenerator::cgUnbox(IRInstruction* inst) {
  SSATmp* dst     = inst->dst();
  SSATmp* src     = inst->src(0);
  auto dstValReg  = m_regs[dst].reg(0);
  auto dstTypeReg = m_regs[dst].reg(1);
  auto srcValReg  = m_regs[src].reg(0);
  auto srcTypeReg = m_regs[src].reg(1);

  assert(dstValReg != dstTypeReg);
  assert(src->type().equals(Type::Gen));
  assert(dst->type().notBoxed());

  emitCmpTVType(m_as, HPHP::KindOfRef, srcTypeReg);
  ifThenElse(CC_E, [&] {
    // srcTypeReg == KindOfRef; srcValReg is RefData*
    const size_t ref_tv_off = RefData::tvOffset();
    if (dstValReg != srcValReg) {
      emitLoadReg(m_as, srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
      emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
                     r32(dstTypeReg));
    } else {
      emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
                     r32(dstTypeReg));
      m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
    }
  }, [&] {
    // srcTypeReg != KindOfRef; copy src -> dst
    shuffle2(m_as, srcValReg, srcTypeReg, dstValReg, dstTypeReg);
  });
}

void CodeGenerator::cgLdFuncCachedCommon(IRInstruction* inst) {
  SSATmp* dst        = inst->dst();
  SSATmp* methodName = inst->src(0);

  const StringData* name = methodName->getValStr();
  CacheHandle ch = TargetCache::allocFixedFunction(name);
  size_t funcCacheOff = ch + offsetof(FixedFuncCache, m_func);

  auto dstReg = m_regs[dst].reg();
  if (dstReg == InvalidReg) {
    // happens if LdFixedFunc and FCall not in same trace
    m_as.   cmpq(0, rVmTl[funcCacheOff]);
  } else {
    m_as.   loadq (rVmTl[funcCacheOff], dstReg);
    m_as.   testq (dstReg, dstReg);
  }
}

void CodeGenerator::cgLdFuncCached(IRInstruction* inst) {
  cgLdFuncCachedCommon(inst);
  // jz off to the helper call in astubs
  unlikelyIfBlock(CC_Z, [&] (Asm& a) {
    // this helper tries the autoload map, and fatals on failure
    cgCallNative(a, inst);
  });
}

void CodeGenerator::cgLdFuncCachedSafe(IRInstruction* inst) {
  cgLdFuncCachedCommon(inst);
  if (Block* taken = inst->taken()) {
    emitFwdJcc(m_as, CC_Z, taken);
  }
}

void CodeGenerator::cgLdFunc(IRInstruction* inst) {
  SSATmp*        dst = inst->dst();
  SSATmp* methodName = inst->src(0);

  TargetCache::CacheHandle ch = TargetCache::FuncCache::alloc();
  // raises an error if function not found
  cgCallHelper(m_as, (TCA)FuncCache::lookup, m_regs[dst].reg(), kSyncPoint,
               ArgGroup(m_regs).imm(ch).ssa(methodName));
}

static void emitLdObjClass(CodeGenerator::Asm& a,
                           PhysReg objReg,
                           PhysReg dstReg) {
  a.loadq  (objReg[ObjectData::getVMClassOffset()], dstReg);
}

void CodeGenerator::cgLdObjClass(IRInstruction* inst) {
  auto dstReg = m_regs[inst->dst()].reg();
  auto objReg = m_regs[inst->src(0)].reg();

  emitLdObjClass(m_as, objReg, dstReg);
}

void CodeGenerator::cgLdObjMethod(IRInstruction *inst) {
  auto cls       = inst->src(0);
  auto clsReg    = m_regs[cls].reg();
  auto name      = inst->src(1);
  auto actRec    = inst->src(2);
  auto actRecReg = m_regs[actRec].reg();
  CacheHandle handle = Transl::TargetCache::MethodCache::alloc();

  // lookup in the targetcache
  assert(MethodCache::kNumLines == 1);
  if (debug) {
    MethodCache::Pair p;
    static_assert(sizeof(p.m_value) == 8,
                  "MethodCache::Pair::m_value assumed to be 8 bytes");
    static_assert(sizeof(p.m_key) == 8,
                  "MethodCache::Pair::m_key assumed to be 8 bytes");
  }

  // preload handle->m_value
  m_as.loadq(rVmTl[handle + offsetof(MethodCache::Pair, m_value)], m_rScratch);
  m_as.cmpq (rVmTl[handle + offsetof(MethodCache::Pair, m_key)], clsReg);
  ifThenElse(CC_E, // if handle->key == cls
             [&] { // then actReg->m_func = handle->value
               m_as.storeq(m_rScratch, actRecReg[AROFF(m_func)]);
             },
             [&] { // else call slow path helper
               cgCallHelper(m_as, (TCA)methodCacheSlowPath, InvalidReg,
                            kSyncPoint,
                            ArgGroup(m_regs).addr(rVmTl, handle)
                                            .ssa(actRec)
                                            .ssa(name)
                                            .ssa(cls));
             });
}

void CodeGenerator::cgStRetVal(IRInstruction* inst) {
  auto  const rFp = m_regs[inst->src(0)].reg();
  auto* const val = inst->src(1);
  cgStore(rFp, AROFF(m_r), val);
}

void CodeGenerator::cgRetAdjustStack(IRInstruction* inst) {
  auto const rFp   = m_regs[inst->src(0)].reg();
  auto const dstSp = m_regs[inst->dst()].reg();
  auto& a = m_as;
  a.    lea   (rFp[AROFF(m_r)], dstSp);
}

void CodeGenerator::cgLdRetAddr(IRInstruction* inst) {
  auto fpReg = m_regs[inst->src(0)].reg(0);
  assert(fpReg != InvalidReg);
  m_as.push(fpReg[AROFF(m_savedRip)]);
}

void checkFrame(ActRec* fp, Cell* sp, bool checkLocals) {
  const Func* func = fp->m_func;
  if (fp->hasVarEnv()) {
    assert(fp->getVarEnv()->getCfp() == fp);
  }
  // TODO: validate this pointer from actrec
  int numLocals = func->numLocals();
  assert(sp <= (Cell*)fp - func->numSlotsInFrame()
         || func->isGenerator());
  if (checkLocals) {
    int numParams = func->numParams();
    for (int i=0; i < numLocals; i++) {
      if (i >= numParams && func->isGenerator() && i < func->numNamedLocals()) {
        continue;
      }
      assert(checkTv(frame_local(fp, i)));
    }
  }
  // We unfortunately can't do the same kind of check for the stack
  // without knowing about FPI regions, because it may contain
  // ActRecs.
}

void traceRet(ActRec* fp, Cell* sp, void* rip) {
  if (rip == TranslatorX64::Get()->getCallToExit()) {
    return;
  }
  checkFrame(fp, sp, /*checkLocals*/ false);
  assert(sp <= (Cell*)fp || fp->m_func->isGenerator());
  // check return value if stack not empty
  if (sp < (Cell*)fp) assertTv(sp);
}

void CodeGenerator::emitTraceRet(CodeGenerator::Asm& a) {
  // call to a trace function
  a.    movq  (rVmFp, rdi);
  a.    movq  (rVmSp, rsi);
  a.    loadq (*rsp, rdx); // return ip from native stack
  // do the call; may use a trampoline
  m_tx64->emitCall(a, TCA(traceRet));
}

void CodeGenerator::cgRetCtrl(IRInstruction* inst) {
  SSATmp* sp = inst->src(0);
  SSATmp* fp = inst->src(1);

  // Make sure rVmFp and rVmSp are set appropriately
  emitMovRegReg(m_as, m_regs[sp].reg(), rVmSp);
  emitMovRegReg(m_as, m_regs[fp].reg(), rVmFp);

  // Return control to caller
  if (RuntimeOption::EvalHHIRGenerateAsserts) {
    emitTraceRet(m_as);
  }
  m_as.ret();
  if (RuntimeOption::EvalHHIRGenerateAsserts) {
    m_as.ud2();
  }
}

void CodeGenerator::emitReqBindAddr(const Func* func,
                                    TCA& dest,
                                    Offset offset) {
  dest = m_tx64->emitServiceReq(SRFlags::None,
                                REQ_BIND_ADDR,
                                2ull,
                                &dest,
                                offset);
}

void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) {
  JmpSwitchData* data = inst->extra<JmpSwitchDest>();
  SSATmp* index       = inst->src(0);
  auto indexReg       = m_regs[index].reg();

  if (!index->isConst()) {
    if (data->bounded) {
      if (data->base) {
        m_as.  subq(data->base, indexReg);
      }
      m_as.    cmpq(data->cases - 2, indexReg);
      prepareForSmash(m_as, kJmpccLen);
      TCA def = m_tx64->emitServiceReq(REQ_BIND_JMPCC_SECOND, 3,
                                   m_as.code.frontier, data->defaultOff, CC_AE);
      m_as.    jae(def);
    }

    TCA* table = m_tx64->allocData<TCA>(sizeof(TCA), data->cases);
    TCA afterLea = m_as.code.frontier + kLeaRipLen;
    ptrdiff_t diff = (TCA)table - afterLea;
    assert(deltaFits(diff, sz::dword));
    m_as.   lea(rip[diff], m_rScratch);
    assert(m_as.code.frontier == afterLea);
    m_as.   jmp(m_rScratch[indexReg*8]);

    for (int i = 0; i < data->cases; i++) {
      emitReqBindAddr(data->func, table[i], data->targets[i]);
    }
  } else {
    int64_t indexVal = index->getValInt();

    if (data->bounded) {
      indexVal -= data->base;
      if (indexVal >= data->cases - 2 || indexVal < 0) {
        m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->defaultOff));
        return;
      }
    }
    m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->targets[indexVal]));
  }
}

typedef FixedStringMap<TCA,true> SSwitchMap;

static TCA sswitchHelperFast(const StringData* val,
                             const SSwitchMap* table,
                             TCA* def) {
  TCA* dest = table->find(val);
  return dest ? *dest : *def;
}

void CodeGenerator::cgLdSSwitchDestFast(IRInstruction* inst) {
  auto data = inst->extra<LdSSwitchDestFast>();

  auto table = m_tx64->allocData<SSwitchMap>(64);
  table->init(data->numCases);
  for (int64_t i = 0; i < data->numCases; ++i) {
    table->add(data->cases[i].str, nullptr);
    TCA* addr = table->find(data->cases[i].str);
    emitReqBindAddr(data->func, *addr, data->cases[i].dest);
  }
  TCA* def = m_tx64->allocData<TCA>(sizeof(TCA), 1);
  emitReqBindAddr(data->func, *def, data->defaultOff);

  cgCallHelper(m_as,
               TCA(sswitchHelperFast),
               inst->dst(),
               kNoSyncPoint,
               ArgGroup(m_regs)
                 .ssa(inst->src(0))
                 .immPtr(table)
                 .immPtr(def));
}

static TCA sswitchHelperSlow(TypedValue typedVal,
                             const StringData** strs,
                             int numStrs,
                             TCA* jmptab) {
  TypedValue* cell = tvToCell(&typedVal);
  for (int i = 0; i < numStrs; ++i) {
    if (tvAsCVarRef(cell).equal(strs[i])) return jmptab[i];
  }
  return jmptab[numStrs]; // default case
}

void CodeGenerator::cgLdSSwitchDestSlow(IRInstruction* inst) {
  auto data = inst->extra<LdSSwitchDestSlow>();

  auto strtab = m_tx64->allocData<const StringData*>(
    sizeof(const StringData*), data->numCases);
  auto jmptab = m_tx64->allocData<TCA>(sizeof(TCA), data->numCases + 1);
  for (int i = 0; i < data->numCases; ++i) {
    strtab[i] = data->cases[i].str;
    emitReqBindAddr(data->func, jmptab[i], data->cases[i].dest);
  }
  emitReqBindAddr(data->func, jmptab[data->numCases], data->defaultOff);

  cgCallHelper(m_as,
               TCA(sswitchHelperSlow),
               inst->dst(),
               kSyncPoint,
               ArgGroup(m_regs)
                 .typedValue(inst->src(0))
                 .immPtr(strtab)
                 .imm(data->numCases)
                 .immPtr(jmptab));
}

/*
 * It'd be nice not to have the cgMov here (and just copy propagate
 * the source or something), but for now we're keeping it allocated to
 * rVmFp so inlined calls to C++ helpers that use the rbp chain to
 * find the caller's ActRec will work correctly.
 *
 * This instruction primarily exists to assist in optimizing away
 * unused activation records, so it's usually not going to happen
 * anyway.
 */
void CodeGenerator::cgDefInlineFP(IRInstruction* inst) {
  auto const fp       = m_regs[inst->src(0)].reg();
  auto const fakeRet  = m_tx64->getRetFromInlinedFrame();
  auto const retBCOff = inst->extra<DefInlineFP>()->retBCOff;

  m_as.    storeq (fakeRet, fp[AROFF(m_savedRip)]);
  m_as.    storel (retBCOff, fp[AROFF(m_soff)]);

  cgMov(inst);
}

void CodeGenerator::cgInlineReturn(IRInstruction* inst) {
  auto fpReg = m_regs[inst->src(0)].reg();
  assert(fpReg == rVmFp);
  m_as.    loadq  (fpReg[AROFF(m_savedRbp)], rVmFp);
}

void CodeGenerator::cgReDefSP(IRInstruction* inst) {
  // TODO(#2288359): this instruction won't be necessary (for
  // non-generator frames) when we don't track rVmSp independently
  // from rVmFp.  In generator frames we'll have to track offsets from
  // a DefGeneratorSP or something similar.
  auto fp  = m_regs[inst->src(0)].reg();
  auto dst = m_regs[inst->dst()].reg();
  auto off = -inst->extra<ReDefSP>()->offset * sizeof(Cell);
  emitLea(m_as, fp[off], dst);
}

void CodeGenerator::cgStashGeneratorSP(IRInstruction* inst) {
  cgMov(inst);
}

void CodeGenerator::cgReDefGeneratorSP(IRInstruction* inst) {
  cgMov(inst);
}

void CodeGenerator::cgFreeActRec(IRInstruction* inst) {
  m_as.loadq(m_regs[inst->src(0)].reg()[AROFF(m_savedRbp)],
             m_regs[inst->dst()].reg());
}

void CodeGenerator::cgSpill(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* src   = inst->src(0);

  assert(dst->numNeededRegs() == src->numNeededRegs());
  for (int locIndex = 0; locIndex < m_regs[src].numAllocatedRegs();
       ++locIndex) {
    // We do not need to mask booleans, since the IR will reload the spill
    auto srcReg = m_regs[src].reg(locIndex);
    auto sinfo = m_regs[dst].spillInfo(locIndex);
    if (m_regs[src].isFullXMM()) {
      m_as.movdqa(srcReg, reg::rsp[sinfo.offset()]);
    } else {
      int offset = sinfo.offset();
      if (locIndex == 0 || packed_tv) {
        emitStoreReg(m_as, srcReg, reg::rsp[offset]);
      } else {
        // Note that type field is shifted in memory
        assert(srcReg.isGP());
        offset += TVOFF(m_type) - (TVOFF(m_data) + sizeof(Value));
        emitStoreTVType(m_as, srcReg, reg::rsp[offset]);
      }
    }
  }
}

void CodeGenerator::cgReload(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* src   = inst->src(0);

  assert(dst->numNeededRegs() == src->numNeededRegs());
  for (int locIndex = 0; locIndex < m_regs[dst].numAllocatedRegs();
       ++locIndex) {
    auto dstReg = m_regs[dst].reg(locIndex);
    auto sinfo = m_regs[src].spillInfo(locIndex);
    if (m_regs[dst].isFullXMM()) {
      assert(dstReg.isXMM());
      m_as.movdqa(reg::rsp[sinfo.offset()], dstReg);
    } else {
      int offset = sinfo.offset();
      if (locIndex == 0 || packed_tv) {
        emitLoadReg(m_as, reg::rsp[offset], dstReg);
      } else {
        // Note that type field is shifted in memory
        offset += TVOFF(m_type) - (TVOFF(m_data) + sizeof(Value));
        assert(dstReg.isGP());
        emitLoadTVType(m_as, reg::rsp[offset], dstReg);
      }
    }
  }
}

void CodeGenerator::cgStPropWork(IRInstruction* inst, bool genTypeStore) {
  SSATmp* obj   = inst->src(0);
  SSATmp* prop  = inst->src(1);
  SSATmp* src   = inst->src(2);
  cgStore(m_regs[obj].reg(), prop->getValInt(), src, genTypeStore);
}
void CodeGenerator::cgStProp(IRInstruction* inst) {
  cgStPropWork(inst, true);
}
void CodeGenerator::cgStPropNT(IRInstruction* inst) {
  cgStPropWork(inst, false);
}

void CodeGenerator::cgStMemWork(IRInstruction* inst, bool genStoreType) {
  SSATmp* addr = inst->src(0);
  SSATmp* offset  = inst->src(1);
  SSATmp* src  = inst->src(2);
  cgStore(m_regs[addr].reg(), offset->getValInt(), src, genStoreType);
}
void CodeGenerator::cgStMem(IRInstruction* inst) {
  cgStMemWork(inst, true);
}
void CodeGenerator::cgStMemNT(IRInstruction* inst) {
  cgStMemWork(inst, false);
}

void CodeGenerator::cgStRefWork(IRInstruction* inst, bool genStoreType) {
  auto destReg = m_regs[inst->dst()].reg();
  auto addrReg = m_regs[inst->src(0)].reg();
  SSATmp* src  = inst->src(1);
  always_assert(!m_regs[src].isFullXMM());
  cgStore(addrReg, RefData::tvOffset(), src, genStoreType);
  if (destReg != InvalidReg)  emitMovRegReg(m_as, addrReg, destReg);
}

void CodeGenerator::cgStRef(IRInstruction* inst) {
  cgStRefWork(inst, true);
}
void CodeGenerator::cgStRefNT(IRInstruction* inst) {
  cgStRefWork(inst, false);
}

static int64_t localOffset(int64_t index) {
  return -cellsToBytes(index + 1);
}

static int64_t localOffset(SSATmp* index) {
  return localOffset(index->getValInt());
}

int CodeGenerator::iterOffset(SSATmp* tmp) {
  const Func* func = curFunc();
  int64_t index = tmp->getValInt();
  return -cellsToBytes(((index + 1) * kNumIterCells + func->numLocals()));
}

int CodeGenerator::iterOffset(uint32_t id) {
  const Func* func = curFunc();
  return -cellsToBytes(((id + 1) * kNumIterCells + func->numLocals()));
}

void CodeGenerator::cgStLoc(IRInstruction* inst) {
  cgStore(m_regs[inst->src(0)].reg(),
          localOffset(inst->extra<StLoc>()->locId),
          inst->src(1),
          true /* store type */);
}

void CodeGenerator::cgStLocNT(IRInstruction* inst) {
  cgStore(m_regs[inst->src(0)].reg(),
          localOffset(inst->extra<StLocNT>()->locId),
          inst->src(1),
          false /* store type */);
}

void CodeGenerator::cgSyncABIRegs(IRInstruction* inst) {
  emitMovRegReg(m_as, m_regs[inst->src(0)].reg(), rVmFp);
  emitMovRegReg(m_as, m_regs[inst->src(1)].reg(), rVmSp);
}

void CodeGenerator::cgReqBindJmp(IRInstruction* inst) {
  m_tx64->emitBindJmp(
    m_as,
    SrcKey(curFunc(), inst->extra<ReqBindJmp>()->offset)
  );
}

static void emitExitNoIRStats(Asm& a,
                              TranslatorX64* tx64,
                              const Func* func,
                              SrcKey dest) {
  if (RuntimeOption::EnableInstructionCounts ||
      HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) {
    Stats::emitInc(a,
                   Stats::opcodeToIRPreStatCounter(
                     Op(*func->unit()->at(dest.offset()))),
                   -1,
                   Transl::CC_None,
                   true);
  }
}

void CodeGenerator::cgReqBindJmpNoIR(IRInstruction* inst) {
  auto const dest = SrcKey(curFunc(),
                           inst->extra<ReqBindJmpNoIR>()->offset);
  emitExitNoIRStats(m_as, m_tx64, curFunc(), dest);
  m_tx64->emitBindJmp(m_as, dest, REQ_BIND_JMP_NO_IR);
}

void CodeGenerator::cgReqRetranslateNoIR(IRInstruction* inst) {
  auto const dest = SrcKey(curFunc(),
                           inst->extra<ReqRetranslateNoIR>()->offset);
  emitExitNoIRStats(m_as, m_tx64, curFunc(), dest);
  m_tx64->emitReqRetransNoIR(m_as, dest);
}

void CodeGenerator::cgReqRetranslate(IRInstruction* inst) {
  auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
  auto const destSR = m_tx64->getSrcRec(destSK);
  m_tx64->emitFallbackUncondJmp(m_as, *destSR);
}

static void emitAssertFlagsNonNegative(CodeGenerator::Asm& as) {
  ifThen(as, CC_NGE, [&] { as.ud2(); });
}

static void emitAssertRefCount(CodeGenerator::Asm& as, PhysReg base) {
  as.cmpl(HPHP::RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
  ifThen(as, CC_NBE, [&] { as.ud2(); });
}

static void emitIncRef(CodeGenerator::Asm& as, PhysReg base) {
  if (RuntimeOption::EvalHHIRGenerateAsserts) {
    emitAssertRefCount(as, base);
  }
  // emit incref
  as.incl(base[FAST_REFCOUNT_OFFSET]);
  if (RuntimeOption::EvalHHIRGenerateAsserts) {
    // Assert that the ref count is greater than zero
    emitAssertFlagsNonNegative(as);
  }
}

void CodeGenerator::cgIncRefWork(Type type, SSATmp* src) {
  assert(type.maybeCounted());
  auto increfMaybeStatic = [&] {
    auto base = m_regs[src].reg(0);
    if (!type.needsStaticBitCheck()) {
      emitIncRef(m_as, base);
    } else {
      m_as.cmpl(RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
      ifThen(m_as, CC_NE, [&] { emitIncRef(m_as, base); });
    }
  };

  if (type.isKnownDataType()) {
    assert(IS_REFCOUNTED_TYPE(type.toDataType()));
    increfMaybeStatic();
  } else {
    emitCmpTVType(m_as, KindOfRefCountThreshold, m_regs[src].reg(1));
    ifThen(m_as, CC_NLE, [&] { increfMaybeStatic(); });
  }
}

void CodeGenerator::cgIncRef(IRInstruction* inst) {
  SSATmp* dst    = inst->dst();
  SSATmp* src    = inst->src(0);
  Type type = src->type();

  cgIncRefWork(type, src);
  shuffle2(m_as, m_regs[src].reg(0), m_regs[src].reg(1),
           m_regs[dst].reg(0), m_regs[dst].reg(1));
}

void CodeGenerator::cgDecRefStack(IRInstruction* inst) {
  cgDecRefMem(inst->typeParam(),
              m_regs[inst->src(0)].reg(),
              cellsToBytes(inst->extra<DecRefStack>()->offset),
              nullptr);
}

void CodeGenerator::cgDecRefThis(IRInstruction* inst) {
  SSATmp* fp    = inst->src(0);
  Block* exit   = inst->taken();
  auto fpReg = m_regs[fp].reg();
  auto scratchReg = m_rScratch;

  // Load AR->m_this into m_rScratch
  m_as.loadq(fpReg[AROFF(m_this)], scratchReg);

  auto decrefIfAvailable = [&] {
    // Check if this is available and we're not in a static context instead
    m_as.testb(1, rbyte(scratchReg));
    ifThen(m_as, CC_Z, [&] {
      cgDecRefStaticType(
        Type::Obj,
        scratchReg,
        exit,
        true /* genZeroCheck */
      );
    });
  };

  if (curFunc()->isPseudoMain()) {
    // In pseudo-mains, emit check for presence of m_this
    m_as.testq(scratchReg, scratchReg);
    ifThen(m_as, CC_NZ, [&] { decrefIfAvailable(); });
  } else {
    decrefIfAvailable();
  }
}

void CodeGenerator::cgDecRefLoc(IRInstruction* inst) {
  cgDecRefMem(inst->typeParam(),
              m_regs[inst->src(0)].reg(),
              localOffset(inst->extra<DecRefLoc>()->locId),
              inst->taken());
}

void CodeGenerator::cgGenericRetDecRefs(IRInstruction* inst) {
  auto const rFp       = m_regs[inst->src(0)].reg();
  auto const numLocals = inst->src(1)->getValInt();
  auto const rDest     = m_regs[inst->dst()].reg();
  auto& a = m_as;

  assert(rFp == rVmFp &&
         "free locals helper assumes the frame pointer is rVmFp");
  assert(rDest == rVmSp &&
         "free locals helper adjusts rVmSp, which must be our dst reg");

  if (numLocals == 0) return;

  // The helpers called below use a special ABI, in which r15 is not saved.
  // So save r15 on the stack if it's live.
  bool saveR15 = m_state.liveRegs[inst].contains(r15);

  int stackAdjust = 8;
  if (saveR15)  {
    a.push(r15);
    stackAdjust = 16;
  }

  auto const target = numLocals > kNumFreeLocalsHelpers
    ? m_tx64->m_freeManyLocalsHelper
    : m_tx64->m_freeLocalsHelpers[numLocals - 1];

  a.subq(stackAdjust, rsp);  // For parity; callee does retq $0x8.
  a.lea(rFp[-numLocals * sizeof(TypedValue)], rDest);
  a.call(target);
  recordSyncPoint(a);

  if (saveR15) {
    a.addq(8, rsp);
    a.pop(r15);
  }
}

static void
tv_release_generic(TypedValue* tv) {
  assert(Transl::tx64->stateIsDirty());
  assert(tv->m_type >= KindOfString && tv->m_type <= KindOfRef);
  g_destructors[typeToDestrIndex(tv->m_type)](tv->m_data.pref);
}

static void
tv_release_typed(RefData* pv, DataType dt) {
  assert(Transl::tx64->stateIsDirty());
  assert(dt >= KindOfString && dt <= KindOfRef);
  g_destructors[typeToDestrIndex(dt)](pv);
}

Address CodeGenerator::getDtorGeneric() {
  return (Address)tv_release_generic;
}

Address CodeGenerator::getDtorTyped() {
  return (Address)tv_release_typed;
}

//
// This method generates code that checks the static bit and jumps if the bit
// is set.  If regIsCount is true, reg contains the _count field. Otherwise,
// it's assumed to contain m_data field.
//
// Return value: the address to be patched with the address to jump to in case
// the static bit is set. If the check is unnecessary, this method retuns NULL.
Address CodeGenerator::cgCheckStaticBit(Type type,
                                        PhysReg reg,
                                        bool regIsCount) {
  if (!type.needsStaticBitCheck()) return NULL;

  if (regIsCount) {
    // reg has the _count value
    m_as.cmp_imm32_reg32(RefCountStaticValue, reg);
  } else {
    // reg has the data pointer
    m_as.cmp_imm32_disp_reg32(RefCountStaticValue, FAST_REFCOUNT_OFFSET, reg);
  }

  Address addrToPatch = m_as.code.frontier;
  m_as.jcc8(CC_E, addrToPatch);
  return addrToPatch;
}


//
// Using the given dataReg, this method generates code that checks the static
// bit out of dataReg, and emits a DecRef if needed.
// NOTE: the flags are left with the result of the DecRef's subtraction,
//       which can then be tested immediately after this.
//
// Return value: the address to be patched if a RefCountedStaticValue check is
//               emitted; NULL otherwise.
//
Address CodeGenerator::cgCheckStaticBitAndDecRef(Type type,
                                                 PhysReg dataReg,
                                                 Block* exit) {
  assert(type.maybeCounted());

  Address patchStaticCheck = nullptr;
  const auto scratchReg = m_rScratch;

  bool canUseScratch = dataReg != scratchReg;

  // TODO: run experiments to check whether the 'if' code sequence
  // is any better than the 'else' branch below; otherwise, always
  // use the 'else' code sequence
  if (type.needsStaticBitCheck() && canUseScratch) {
    // If we need to check for static value, then load the _count into a
    // register to avoid doing two loads. The generated sequence is:
    //
    //     scratchReg = [dataReg + offset(_count)]
    //     if scratchReg == RefCountStaticValue then skip DecRef
    //     scratchReg = scratchReg - 1
    //     ( if exit != NULL, emit:
    //          jz exit
    //     )
    //     [dataReg + offset(_count)] = scratchReg

    if (RuntimeOption::EvalHHIRGenerateAsserts) {
      emitAssertRefCount(m_as, dataReg);
    }
    // Load _count in scratchReg
    m_as.loadl(dataReg[FAST_REFCOUNT_OFFSET], r32(scratchReg));

    // Check for RefCountStaticValue
    patchStaticCheck = cgCheckStaticBit(type, scratchReg,
                                        true /* reg has _count */);

    // Decrement count and store it back in memory.
    // If there's an exit, emit jump to it when _count would get down to 0
    m_as.decq(scratchReg);
    if (exit) {
      emitFwdJcc(CC_E, exit);
    }
    if (RuntimeOption::EvalHHIRGenerateAsserts) {
      // Assert that the ref count is greater than zero
      emitAssertFlagsNonNegative(m_as);
    }
    m_as.store_reg32_disp_reg64(scratchReg, FAST_REFCOUNT_OFFSET, dataReg);

  } else {
    // Can't use scratch reg, so emit code that operates directly in
    // memory. Compared to the sequence above, this will result in one
    // extra load, but it has the advantage of producing a instruction
    // sequence.
    //
    //     ( if needStaticBitCheck, emit :
    //           cmp [dataReg + offset(_count)], RefCountStaticValue
    //           je LabelAfterDecRef
    //     )
    //     ( if exit != NULL, emit:
    //           cmp [dataReg + offset(_count)], 1
    //           jz exit
    //     )
    //     sub [dataReg + offset(_count)], 1

    // If necessary, check for RefCountStaticValue
    patchStaticCheck = cgCheckStaticBit(type, dataReg,
                                        false /* passing dataReg */);

    // If there's an exit, emit jump to it if _count would get down to 0
    if (exit) {
      m_as.cmp_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET, dataReg);
      emitFwdJcc(CC_E, exit);
    }
    if (RuntimeOption::EvalHHIRGenerateAsserts) {
      emitAssertRefCount(m_as, dataReg);
    }

    // Decrement _count
    m_as.decl(dataReg[FAST_REFCOUNT_OFFSET]);

    if (RuntimeOption::EvalHHIRGenerateAsserts) {
      // Assert that the ref count is not less than zero
      emitAssertFlagsNonNegative(m_as);
    }
  }

  return patchStaticCheck;
}


//
// Returns the address to be patched with the address to jump to in case
// the type is not ref-counted.
//
Address CodeGenerator::cgCheckRefCountedType(PhysReg typeReg) {
  emitCmpTVType(m_as, KindOfRefCountThreshold, typeReg);
  Address addrToPatch = m_as.code.frontier;
  m_as.jcc8(CC_LE, addrToPatch);
  return addrToPatch;
}

Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset) {
  emitCmpTVType(m_as, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]);
  Address addrToPatch = m_as.code.frontier;
  m_as.jcc8(CC_LE, addrToPatch);
  return addrToPatch;
}

//
// Generates dec-ref of a typed value with statically known type.
//
void CodeGenerator::cgDecRefStaticType(Type type,
                                       PhysReg dataReg,
                                       Block* exit,
                                       bool genZeroCheck) {
  assert(type != Type::Cell && type != Type::Gen);
  assert(type.isKnownDataType());

  if (type.notCounted()) return;

  // Check for RefCountStaticValue if needed, do the actual DecRef,
  // and leave flags set based on the subtract result, which is
  // tested below
  Address patchStaticCheck;
  if (genZeroCheck) {
    patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, exit);
  } else {
    // Set exit as NULL so that the code doesn't jump to error checking.
    patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, nullptr);
  }

  // If not exiting on count down to zero, emit the zero-check and
  // release call
  if (genZeroCheck && exit == nullptr) {
    // Emit jump to m_astubs (to call release) if count got down to zero
    unlikelyIfBlock(CC_Z, [&] (Asm& a) {
      // Emit the call to release in m_astubs
      cgCallHelper(a, m_tx64->getDtorCall(type.toDataType()),
                   InvalidReg, InvalidReg, kSyncPoint,
                   ArgGroup(m_regs).reg(dataReg));
    });
  }
  if (patchStaticCheck) {
    m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
  }
}

//
// Generates dec-ref of a typed value with dynamic (statically unknown) type,
// when the type is stored in typeReg.
//
void CodeGenerator::cgDecRefDynamicType(PhysReg typeReg,
                                        PhysReg dataReg,
                                        Block* exit,
                                        bool genZeroCheck) {
  // Emit check for ref-counted type
  Address patchTypeCheck = cgCheckRefCountedType(typeReg);

  // Emit check for RefCountStaticValue and the actual DecRef
  Address patchStaticCheck;
  if (genZeroCheck) {
    patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, exit);
  } else {
    patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, nullptr);
  }

  // If not exiting on count down to zero, emit the zero-check and release call
  if (genZeroCheck && exit == nullptr) {
    // Emit jump to m_astubs (to call release) if count got down to zero
    unlikelyIfBlock(CC_Z, [&] (Asm& a) {
      // Emit call to release in m_astubs
      cgCallHelper(a, getDtorTyped(), InvalidReg, kSyncPoint,
                   ArgGroup(m_regs).reg(dataReg).reg(typeReg));
    });
  }
  // Patch checks to jump around the DecRef
  if (patchTypeCheck)   m_as.patchJcc8(patchTypeCheck,   m_as.code.frontier);
  if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
}

//
// Generates dec-ref of a typed value with dynamic (statically
// unknown) type, when all we have is the baseReg and offset of
// the typed value. This method assumes that baseReg is not the
// scratch register.
//
void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
                                           int64_t offset,
                                           Block* exit) {
  auto scratchReg = m_rScratch;

  assert(baseReg != scratchReg);

  // Emit check for ref-counted type
  Address patchTypeCheck = cgCheckRefCountedType(baseReg, offset);
  if (exit == nullptr && RuntimeOption::EvalHHIRGenericDtorHelper) {
    {
      // This PhysRegSaverStub saves rdi redundantly if
      // !liveRegs[m_curInst].contains(rdi), but its
      // necessary to maintain stack alignment. We can do better
      // by making the helpers adjust the stack for us in the cold
      // path, which calls the destructor.
      PhysRegSaverStub regSaver(m_as, RegSet(rdi));

      /*
       * rVmSp is ok here because this is part of the special
       * ABI to m_irPopRHelper.  We're not using a secret dependency
       * on the frame or stack---we're only going to use that ABI if
       * we happen to have that register allocated for baseReg.
       */
      if (offset == 0 && baseReg == rVmSp) {
        // Decref'ing top of vm stack, very likely a popR
        m_tx64->emitCall(m_as, m_tx64->m_irPopRHelper);
      } else {
        if (baseReg == rsp) {
          // Because we just pushed %rdi, %rsp is 8 bytes below where
          // offset is expecting it to be.
          offset += sizeof(int64_t);
        }
        m_as.lea(baseReg[offset], rdi);
        m_tx64->emitCall(m_as, m_tx64->m_dtorGenericStub);
      }
      recordSyncPoint(m_as);
    }
    if (patchTypeCheck) {
      m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
    }
    return;
  }
  // Load m_data into the scratch reg
  m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);

  // Emit check for RefCountStaticValue and the actual DecRef
  Address patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, scratchReg,
                                                       exit);

  // If not exiting on count down to zero, emit the zero-check and release call
  if (exit == nullptr) {
    // Emit jump to m_astubs (to call release) if count got down to zero
    unlikelyIfBlock(CC_Z, [&] (Asm& a) {
      // Emit call to release in m_astubs
      a.lea(baseReg[offset], scratchReg);
      cgCallHelper(a, getDtorGeneric(), InvalidReg, kSyncPoint,
                   ArgGroup(m_regs).reg(scratchReg));
    });
  }

  // Patch checks to jump around the DecRef
  if (patchTypeCheck)   m_as.patchJcc8(patchTypeCheck,   m_as.code.frontier);
  if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
}

//
// Generates the dec-ref of a typed value in memory address [baseReg + offset].
// This handles cases where type is either static or dynamic.
//
void CodeGenerator::cgDecRefMem(Type type,
                                PhysReg baseReg,
                                int64_t offset,
                                Block* exit) {
  auto scratchReg = m_rScratch;
  assert(baseReg != scratchReg);

  if (type.needsReg()) {
    // The type is dynamic, but we don't have two registers available
    // to load the type and the data.
    cgDecRefDynamicTypeMem(baseReg, offset, exit);
  } else if (type.maybeCounted()) {
    m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
    cgDecRefStaticType(type, scratchReg, exit, true);
  }
}

void CodeGenerator::cgDecRefMem(IRInstruction* inst) {
  assert(inst->src(0)->type().isPtr());
  cgDecRefMem(inst->typeParam(),
              m_regs[inst->src(0)].reg(),
              inst->src(1)->getValInt(),
              inst->taken());
}

void CodeGenerator::cgDecRefWork(IRInstruction* inst, bool genZeroCheck) {
  SSATmp* src   = inst->src(0);
  if (!isRefCounted(src)) return;
  Block* exit = inst->taken();
  Type type = src->type();
  if (type.isKnownDataType()) {
    cgDecRefStaticType(type, m_regs[src].reg(), exit, genZeroCheck);
  } else {
    cgDecRefDynamicType(m_regs[src].reg(1),
                        m_regs[src].reg(0),
                        exit,
                        genZeroCheck);
  }
}

void CodeGenerator::cgDecRef(IRInstruction *inst) {
  // DecRef may bring the count to zero, and run the destructor.
  // Generate code for this.
  assert(!inst->taken());
  cgDecRefWork(inst, true);
}

void CodeGenerator::cgDecRefNZ(IRInstruction* inst) {
  // DecRefNZ cannot bring the count to zero.
  // Therefore, we don't generate zero-checking code.
  assert(!inst->taken());
  cgDecRefWork(inst, false);
}

void CodeGenerator::cgDecRefNZOrBranch(IRInstruction* inst) {
  assert(inst->taken());
  cgDecRefWork(inst, true);
}

void CodeGenerator::cgCufIterSpillFrame(IRInstruction* inst) {
  auto const sp    = inst->src(0);
  auto const fp    = inst->src(1);
  auto const nArgs = inst->extra<CufIterSpillFrame>()->args;
  auto const iterId = inst->extra<CufIterSpillFrame>()->iterId;
  auto const itOff = iterOffset(iterId);

  const int64_t spOffset = -kNumActRecCells * sizeof(Cell);
  auto spReg = m_regs[sp].reg();
  auto fpReg = m_regs[fp].reg();

  m_as.loadq   (fpReg[itOff + CufIter::funcOff()], m_rScratch);
  m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_func))]);

  m_as.loadq   (fpReg[itOff + CufIter::ctxOff()], m_rScratch);
  m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_this))]);

  m_as.shrq    (1, m_rScratch);
  ifThen(m_as, CC_NBE, [this] {
      m_as.shlq(1, m_rScratch);
      emitIncRef(m_as, m_rScratch);
    });
  m_as.loadq   (fpReg[itOff + CufIter::nameOff()], m_rScratch);
  m_as.testq    (m_rScratch, m_rScratch);
  ifThen(m_as, CC_NZ, [this] {
      m_as.cmpl(RefCountStaticValue, m_rScratch[FAST_REFCOUNT_OFFSET]);
      ifThen(m_as, CC_NE, [&] { emitIncRef(m_as, m_rScratch); });
      m_as.orq (ActRec::kInvNameBit, m_rScratch);
    });
  m_as.storeq  (m_rScratch, spReg[spOffset + int(AROFF(m_invName))]);
  m_as.storeq  (fpReg, spReg[spOffset + int(AROFF(m_savedRbp))]);
  m_as.storel  (nArgs, spReg[spOffset + int(AROFF(m_numArgsAndCtorFlag))]);

  emitAdjustSp(spReg,
               m_regs[inst->dst()].reg(),
               spOffset);
}

void CodeGenerator::cgSpillFrame(IRInstruction* inst) {
  auto const sp        = inst->src(0);
  auto const fp        = inst->src(1);
  auto const func      = inst->src(2);
  auto const objOrCls  = inst->src(3);
  auto const magicName = inst->extra<SpillFrame>()->invName;
  auto const nArgs     = inst->extra<SpillFrame>()->numArgs;

  const int64_t spOffset = -kNumActRecCells * sizeof(Cell);

  DEBUG_ONLY bool setThis = true;

  auto spReg = m_regs[sp].reg();
  // actRec->m_this
  if (objOrCls->isA(Type::Cls)) {
    // store class
    if (objOrCls->isConst()) {
      m_as.store_imm64_disp_reg64(uintptr_t(objOrCls->getValClass()) | 1,
                                  spOffset + int(AROFF(m_this)),
                                  spReg);
    } else {
      Reg64 clsPtrReg = m_regs[objOrCls].reg();
      m_as.movq  (clsPtrReg, m_rScratch);
      m_as.orq   (1, m_rScratch);
      m_as.storeq(m_rScratch, spReg[spOffset + int(AROFF(m_this))]);
    }
  } else if (objOrCls->isA(Type::Obj)) {
    // store this pointer
    m_as.store_reg64_disp_reg64(m_regs[objOrCls].reg(),
                                spOffset + int(AROFF(m_this)),
                                spReg);
  } else if (objOrCls->isA(Type::Ctx)) {
    // Stores either a this pointer or a Cctx -- statically unknown.
    Reg64 objOrClsPtrReg = m_regs[objOrCls].reg();
    m_as.storeq(objOrClsPtrReg, spReg[spOffset + int(AROFF(m_this))]);
  } else {
    assert(objOrCls->isA(Type::InitNull));
    // no obj or class; this happens in FPushFunc
    int offset_m_this = spOffset + int(AROFF(m_this));
    // When func is either Type::FuncCls or Type::FuncCtx,
    // m_this/m_cls will be initialized below
    if (!func->isConst() && (func->isA(Type::FuncCtx))) {
      // m_this is unioned with m_cls and will be initialized below
      setThis = false;
    } else {
      m_as.store_imm64_disp_reg64(0, offset_m_this, spReg);
    }
  }
  // actRec->m_invName
  // ActRec::m_invName is encoded as a pointer with bit kInvNameBit
  // set to distinguish it from m_varEnv and m_extrArgs
  uintptr_t invName = !magicName
    ? 0
    : reinterpret_cast<uintptr_t>(magicName) | ActRec::kInvNameBit;
    m_as.store_imm64_disp_reg64(invName,
                                spOffset + int(AROFF(m_invName)),
                                spReg);
  // actRec->m_func  and possibly actRec->m_cls
  // Note m_cls is unioned with m_this and may overwrite previous value
  if (func->type().isNull()) {
    assert(func->isConst());
  } else if (func->isConst()) {
    const Func* f = func->getValFunc();
    m_as. mov_imm64_reg((uint64_t)f, m_rScratch);
    m_as.store_reg64_disp_reg64(m_rScratch,
                                spOffset + int(AROFF(m_func)),
                                spReg);
    if (func->isA(Type::FuncCtx)) {
      // Fill in m_cls if provided with both func* and class*
      CG_PUNT(cgAllocActRec);
    }
  } else {
    int offset_m_func = spOffset + int(AROFF(m_func));
    m_as.store_reg64_disp_reg64(m_regs[func].reg(0),
                                offset_m_func,
                                spReg);
    if (func->isA(Type::FuncCtx)) {
      int offset_m_cls = spOffset + int(AROFF(m_cls));
      m_as.store_reg64_disp_reg64(m_regs[func].reg(1),
                                  offset_m_cls,
                                  spReg);
      setThis = true; /* m_this and m_cls are in a union */
    }
  }
  assert(setThis);
  // actRec->m_savedRbp
  m_as.store_reg64_disp_reg64(m_regs[fp].reg(),
                              spOffset + int(AROFF(m_savedRbp)),
                              spReg);

  // actRec->m_numArgsAndCtorFlag
  m_as.store_imm32_disp_reg(nArgs,
                            spOffset + int(AROFF(m_numArgsAndCtorFlag)),
                            spReg);

  emitAdjustSp(spReg,
               m_regs[inst->dst()].reg(),
               spOffset);
}

const Func* loadClassCtor(Class* cls) {
  const Func* f = cls->getCtor();
  if (UNLIKELY(!(f->attrs() & AttrPublic))) {
    VMRegAnchor _;
    UNUSED MethodLookup::LookupResult res =
      g_vmContext->lookupCtorMethod(f, cls, true /*raise*/);
    assert(res == MethodLookup::MethodFoundWithThis);
  }
  return f;
}

Instance* createClHelper(Class* cls, int numArgs, ActRec* ar, TypedValue* sp) {
  Instance* newObj = newInstance(cls);
  newObj->incRefCount();
  return static_cast<c_Closure*>(newObj)->init(numArgs, ar, sp);
}

void CodeGenerator::cgAllocObjFast(IRInstruction* inst) {
  const Class* cls = inst->src(0)->getValClass();
  auto dstReg = m_regs[inst->dst()].reg();

  // First, make sure our property init vectors are all set up
  bool props = cls->pinitVec().size() > 0;
  bool sprops = cls->numStaticProperties() > 0;
  assert((props || sprops) == cls->needInitialization());
  if (cls->needInitialization()) {
    if (props) {
      cls->initPropHandle();
      m_as.testq(-1, rVmTl[cls->propHandle()]);
      unlikelyIfBlock(CC_Z, [&] (Asm& a) {
          cgCallHelper(a,
                       (TCA)getMethodPtr(&Class::initProps),
                       InvalidReg,
                       kSyncPoint,
                       ArgGroup(m_regs).imm((uint64_t)cls));
      });
    }
    if (sprops) {
      cls->initSPropHandle();
      m_as.testq(-1, rVmTl[cls->sPropHandle()]);
      unlikelyIfBlock(CC_Z, [&] (Asm& a) {
          cgCallHelper(a,
                       (TCA)getMethodPtr(&Class::initSProps),
                       InvalidReg,
                       kSyncPoint,
                       ArgGroup(m_regs).imm((uint64_t)cls));
      });
    }
  }

  // Next, allocate the object
  if (cls->instanceCtor()) {
    cgCallHelper(m_as,
                 (TCA)cls->instanceCtor(),
                 dstReg,
                 kSyncPoint,
                 ArgGroup(m_regs).imm((uint64_t)cls));
  } else {
    size_t size = Instance::sizeForNProps(cls->numDeclProperties());
    int allocator = object_alloc_size_to_index(size);
    assert(allocator != -1);
    cgCallHelper(m_as,
                 (TCA)getMethodPtr(&Instance::newInstanceRaw),
                 dstReg,
                 kSyncPoint,
                 ArgGroup(m_regs).imm((uint64_t)cls).imm(allocator));
  }

  // Set the attributes, if any
  int odAttrs = cls->getODAttrs();
  if (odAttrs) {
    // o_attribute is 16 bits but the fact that we're or-ing a mask makes it ok
    assert(!(odAttrs & 0xffff0000));
    m_as.orq(odAttrs, dstReg[ObjectData::attributeOff()]);
  }

  // Initialize the properties
  size_t nProps = cls->numDeclProperties();
  if (nProps > 0) {
    m_as.push(dstReg);
    m_as.subq(8, reg::rsp);
    if (cls->pinitVec().size() == 0) {
      // Fast case: copy from a known address in the Class
      ArgGroup args = ArgGroup(m_regs)
        .addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
        .imm(int64_t(&cls->declPropInit()[0]))
        .imm(cellsToBytes(nProps));
      cgCallHelper(m_as,
                   (TCA)memcpy,
                   InvalidReg,
                   kNoSyncPoint,
                   args);
    } else {
      // Slower case: we have to load the src address from the targetcache
      auto rPropData = m_rScratch;
      // Load the Class's propInitVec from the targetcache
      m_as.loadq(rVmTl[cls->propHandle()], rPropData);
      // propData holds the PropInitVec. We want &(*propData)[0]
      m_as.loadq(rPropData[Class::PropInitVec::dataOff()], rPropData);
      if (!cls->hasDeepInitProps()) {
        ArgGroup args = ArgGroup(m_regs)
          .addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
          .reg(rPropData)
          .imm(cellsToBytes(nProps));
        cgCallHelper(m_as,
                     (TCA)memcpy,
                     InvalidReg,
                     kNoSyncPoint,
                     args);
      } else {
        ArgGroup args = ArgGroup(m_regs)
          .addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
          .reg(rPropData)
          .imm(nProps);
        cgCallHelper(m_as,
                     (TCA)deepInitHelper,
                     InvalidReg,
                     kNoSyncPoint,
                     args);
      }
    }
    m_as.addq(8, reg::rsp);
    m_as.pop(dstReg);
  }
  if (cls->callsCustomInstanceInit()) {
    // callCustomInstanceInit returns the instance in rax
    cgCallHelper(m_as,
                 (TCA)getMethodPtr(&Instance::callCustomInstanceInit),
                 dstReg,
                 kSyncPoint,
                 ArgGroup(m_regs).reg(dstReg));
  }
}

void CodeGenerator::cgInlineCreateCont(IRInstruction* inst) {
  auto const& data = *inst->extra<InlineCreateCont>();
  auto const helper = data.origFunc->isMethod()
    ? &VMExecutionContext::createContinuationHelper<true>
    : &VMExecutionContext::createContinuationHelper<false>;

  cgCallHelper(
    m_as,
    reinterpret_cast<TCA>(helper),
    inst->dst(),
    kSyncPoint,
    ArgGroup(m_regs)
      .immPtr(data.origFunc)
      .immPtr(data.genFunc)
      .ssa(inst->src(0))
      // Deliberately ignoring frameStaticClass parameter, because
      // it's unused if we have a $this pointer, and we don't inline
      // functions with a null $this.
  );
  if (data.origFunc->isMethod()) {
    // We can't support a null $this.
    assert(inst->src(0)->isA(Type::Obj));
  }
}

void CodeGenerator::cgCallArray(IRInstruction* inst) {
  Offset pc             = inst->extra<CallArray>()->pc;
  Offset after          = inst->extra<CallArray>()->after;

  ArgGroup args(m_regs);
  args.imm(pc).imm(after);

  // fCallArrayHelper makes the actual call by smashing its return address.
  cgCallHelper(m_as, (TCA)TranslatorX64::fCallArrayHelper,
               nullptr, kSyncPoint, args);
}

void CodeGenerator::cgCall(IRInstruction* inst) {
  SSATmp* actRec         = inst->src(0);
  SSATmp* returnBcOffset = inst->src(1);
  SSATmp* func           = inst->src(2);
  SrcRange args          = inst->srcs().subpiece(3);
  int32_t numArgs        = args.size();

  auto spReg = m_regs[actRec].reg();
  // put all outgoing arguments onto the VM stack
  int64_t adjustment = (-(int64_t)numArgs) * sizeof(Cell);
  for (int32_t i = 0; i < numArgs; i++) {
    // Type::None here means that the simplifier proved that the value
    // matches the value already in memory, thus the store is redundant.
    if (args[i]->type() != Type::None) {
      cgStore(spReg, -(i + 1) * sizeof(Cell), args[i]);
    }
  }
  // store the return bytecode offset into the outgoing actrec
  uint64_t returnBc = returnBcOffset->getValInt();
  m_as.store_imm32_disp_reg(returnBc, AROFF(m_soff), spReg);
  if (adjustment != 0) {
    m_as.add_imm32_reg64(adjustment, spReg);
  }

  assert(m_state.lastMarker);
  SrcKey srcKey = SrcKey(m_state.lastMarker->func, m_state.lastMarker->bcOff);
  bool isImmutable = (func->isConst() && !func->type().isNull());
  const Func* funcd = isImmutable ? func->getValFunc() : nullptr;
  assert(&m_as == &m_tx64->getAsm());
  int32_t adjust = m_tx64->emitBindCall(srcKey, funcd, numArgs);
  if (adjust) {
    m_as.addq (adjust, rVmSp);
  }
}

void CodeGenerator::cgCastStk(IRInstruction *inst) {
  Type type       = inst->typeParam();
  SSATmp* sp      = inst->src(0);
  uint32_t offset = inst->extra<CastStk>()->offset;
  PhysReg spReg   = m_regs[sp].reg();

  ArgGroup args(m_regs);
  args.addr(spReg, cellsToBytes(offset));

  TCA tvCastHelper;
  if (type.subtypeOf(Type::Bool)) {
    tvCastHelper = (TCA)tvCastToBooleanInPlace;
  } else if (type.subtypeOf(Type::Int)) {
    // if casting to integer, pass 10 as the base for the conversion
    args.imm(10);
    tvCastHelper = (TCA)tvCastToInt64InPlace;
  } else if (type.subtypeOf(Type::Dbl)) {
    tvCastHelper = (TCA)tvCastToDoubleInPlace;
  } else if (type.subtypeOf(Type::Arr)) {
    tvCastHelper = (TCA)tvCastToArrayInPlace;
  } else if (type.subtypeOf(Type::Str)) {
    tvCastHelper = (TCA)tvCastToStringInPlace;
  } else if (type.subtypeOf(Type::Obj)) {
    tvCastHelper = (TCA)tvCastToObjectInPlace;
  } else {
    not_reached();
  }
  cgCallHelper(m_as, tvCastHelper, nullptr,
               kSyncPoint, args, DestType::None);
}

void CodeGenerator::cgCallBuiltin(IRInstruction* inst) {
  SSATmp* f             = inst->src(0);
  auto args             = inst->srcs().subpiece(2);
  int32_t numArgs       = args.size();
  SSATmp* dst           = inst->dst();
  auto dstReg           = m_regs[dst].reg(0);
  auto dstType          = m_regs[dst].reg(1);
  Type returnType       = inst->typeParam();

  const Func* func = f->getValFunc();
  DataType funcReturnType = func->returnType();
  int returnOffset = HHIR_MISOFF(tvBuiltinReturn);

  if (TranslatorX64::eagerRecord(func)) {
    const uchar* pc = curUnit()->entry() + m_state.lastMarker->bcOff;
    // we have spilled all args to stack, so spDiff is 0
    m_tx64->emitEagerSyncPoint(m_as, pc, 0);
  }
  // RSP points to the MInstrState we need to use.
  // workaround the fact that rsp moves when we spill registers around call
  PhysReg misReg = m_rScratch;
  emitMovRegReg(m_as, reg::rsp, misReg);

  ArgGroup callArgs(m_regs);
  if (isCppByRef(funcReturnType)) {
    // first arg is pointer to storage for that return value
    if (isSmartPtrRef(funcReturnType)) {
      returnOffset += TVOFF(m_data);
    }
    // misReg is pointing to an MInstrState struct on the C stack.  Pass
    // the address of tvBuiltinReturn to the native function as the location
    // it can construct the return Array, String, Object, or Variant.
    callArgs.addr(misReg, returnOffset); // &misReg[returnOffset]
  }

  // non-pointer args are plain values passed by value.  String, Array,
  // Object, and Variant are passed by const&, ie a pointer to stack memory
  // holding the value, so expect PtrToT types for these.
  // Pointers to smartptr types (String, Array, Object) need adjusting to
  // point to &ptr->m_data.
  for (int i = 0; i < numArgs; i++) {
    const Func::ParamInfo& pi = func->params()[i];
    if (TVOFF(m_data) && isSmartPtrRef(pi.builtinType())) {
      assert(args[i]->type().isPtr() && m_regs[args[i]].reg() != InvalidReg);
      callArgs.addr(m_regs[args[i]].reg(), TVOFF(m_data));
    } else {
      callArgs.ssa(args[i]);
    }
  }

  // if the return value is returned by reference, we don't need the
  // return value from this call since we know where the value is.
  cgCallHelper(m_as, Transl::CppCall((TCA)func->nativeFuncPtr()),
               isCppByRef(funcReturnType) ? InvalidReg : dstReg,
               kSyncPoint, callArgs);

  // load return value from builtin
  // for primitive return types (int, bool), the return value
  // is already in dstReg (the builtin call returns in rax). For return
  // by reference (String, Object, Array, Variant), the builtin writes the
  // return value into MInstrState::tvBuiltinReturn TV, from where it
  // has to be tested and copied.
  if (dstReg == InvalidReg || returnType.isSimpleType()) {
    return;
  }
  // after the call, RSP is back pointing to MInstrState and rSratch
  // has been clobberred.
  misReg = rsp;

  if (returnType.isReferenceType()) {
    assert(isCppByRef(funcReturnType) && isSmartPtrRef(funcReturnType));
    // return type is String, Array, or Object; fold nullptr to KindOfNull
    m_as.   loadq (misReg[returnOffset], dstReg);
    emitLoadImm(m_as, returnType.toDataType(), dstType);
    emitLoadImm(m_as, KindOfNull, m_rScratch);
    m_as.   testq (dstReg, dstReg);
    m_as.   cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
    return;
  }
  if (returnType.subtypeOf(Type::Cell)
      || returnType.subtypeOf(Type::BoxedCell)) {
    // return type is Variant; fold KindOfUninit to KindOfNull
    assert(isCppByRef(funcReturnType) && !isSmartPtrRef(funcReturnType));
    assert(misReg != dstType);
    emitLoadTVType(m_as, misReg[returnOffset + TVOFF(m_type)], dstType);
    m_as.   loadq (misReg[returnOffset + TVOFF(m_data)], dstReg);
    emitLoadImm(m_as, KindOfNull, m_rScratch);
    static_assert(KindOfUninit == 0, "KindOfUninit must be 0 for test");
    m_as.   testb (rbyte(dstType), rbyte(dstType));
    m_as.   cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
    return;
  }
  not_reached();
}

void CodeGenerator::cgSpillStack(IRInstruction* inst) {
  SSATmp* dst             = inst->dst();
  SSATmp* sp              = inst->src(0);
  auto const spDeficit    = inst->src(1)->getValInt();
  auto const spillVals    = inst->srcs().subpiece(2);
  auto const numSpillSrcs = spillVals.size();
  auto const dstReg       = m_regs[dst].reg();
  auto const spReg        = m_regs[sp].reg();
  auto const spillCells   = spillValueCells(inst);

  int64_t adjustment = (spDeficit - spillCells) * sizeof(Cell);
  for (uint32_t i = 0; i < numSpillSrcs; ++i) {
    const int64_t offset = i * sizeof(Cell) + adjustment;
    if (spillVals[i]->type() == Type::None) {
      // The simplifier detected that we're storing the same value
      // already in there.
      continue;
    }

    auto* val = spillVals[i];
    auto* inst = val->inst();
    while (inst->isPassthrough()) {
      inst = inst->getPassthroughValue()->inst();
    }
    // If our value came from a LdStack on the same sp and offset,
    // we don't need to spill it.
    if (inst->op() == LdStack && inst->src(0) == sp &&
        inst->extra<LdStack>()->offset * sizeof(Cell) == offset) {
      FTRACE(6, "{}: Not spilling spill value {} from {}\n",
             __func__, i, inst->toString());
    } else {
      cgStore(spReg, offset, val);
    }
  }

  emitAdjustSp(spReg, dstReg, adjustment);
}

void CodeGenerator::emitAdjustSp(PhysReg spReg,
                                 PhysReg dstReg,
                                 int64_t adjustment /* bytes */) {
  if (adjustment != 0) {
    if (dstReg != spReg) {
      m_as.    lea   (spReg[adjustment], dstReg);
    } else {
      m_as.    addq  (adjustment, dstReg);
    }
  } else {
    emitMovRegReg(m_as, spReg, dstReg);
  }
}

void CodeGenerator::cgNativeImpl(IRInstruction* inst) {
  SSATmp* func  = inst->src(0);
  SSATmp* fp    = inst->src(1);

  assert(func->isConst());
  assert(func->type() == Type::Func);
  const Func* fn = func->getValFunc();

  BuiltinFunction builtinFuncPtr = func->getValFunc()->builtinFuncPtr();
  emitMovRegReg(m_as, m_regs[fp].reg(), argNumToRegName[0]);
  if (TranslatorX64::eagerRecord(fn)) {
    m_tx64->emitEagerSyncPoint(m_as, fn->getEntry(), 0);
  }
  m_as.call((TCA)builtinFuncPtr);
  recordSyncPoint(m_as);
}

void CodeGenerator::cgLdThis(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* src   = inst->src(0);
  Block* label  = inst->taken();
  // mov dst, [fp + 0x20]
  auto dstReg = m_regs[dst].reg();

  // the destination of LdThis could be dead but the instruction
  // itself still useful because of the checks that it does (if it has
  // a label).  So we need to make sure there is a dstReg for this
  // instruction.
  if (dstReg != InvalidReg) {
    // instruction's result is not dead
    m_as.loadq(m_regs[src].reg()[AROFF(m_this)], dstReg);
  }
  if (label == NULL) return;  // no need to perform its checks
  if (dstReg != InvalidReg) {
    // test 0x01, dst
    m_as.testb(1, rbyte(dstReg));
  } else {
    m_as.testb(1, m_regs[src].reg()[AROFF(m_this)]);
  }
  // jnz label
  emitFwdJcc(CC_NZ, label);
}

static void emitLdClsCctx(CodeGenerator::Asm& a,
                          PhysReg srcReg,
                          PhysReg dstReg) {
  emitMovRegReg(a, srcReg, dstReg);
  a.    decq(dstReg);
}

void CodeGenerator::cgLdClsCtx(IRInstruction* inst) {
  PhysReg srcReg = m_regs[inst->src(0)].reg();
  PhysReg dstReg = m_regs[inst->dst()].reg();
  // Context could be either a this object or a class ptr
  m_as.   testb(1, rbyte(srcReg));
  ifThenElse(CC_NZ,
             [&] { emitLdClsCctx(m_as, srcReg, dstReg);  }, // ctx is a class
             [&] { emitLdObjClass(m_as, srcReg, dstReg); }  // ctx is this ptr
            );
}

void CodeGenerator::cgLdClsCctx(IRInstruction* inst) {
  PhysReg srcReg = m_regs[inst->src(0)].reg();
  PhysReg dstReg = m_regs[inst->dst()].reg();
  emitLdClsCctx(m_as, srcReg, dstReg);
}

void CodeGenerator::cgLdCtx(IRInstruction* inst) {
  PhysReg dstReg = m_regs[inst->dst()].reg();
  PhysReg srcReg = m_regs[inst->src(0)].reg();
  if (dstReg != InvalidReg) {
    m_as.loadq(srcReg[AROFF(m_this)], dstReg);
  }
}

void CodeGenerator::cgLdCctx(IRInstruction* inst) {
  return cgLdCtx(inst);
}

void CodeGenerator::cgLdConst(IRInstruction* inst) {
  auto const dstReg   = m_regs[inst->dst()].reg();
  auto const val      = inst->extra<LdConst>()->as<uintptr_t>();
  if (dstReg == InvalidReg) return;
  emitLoadImm(m_as, val, dstReg);
}

void CodeGenerator::cgLdARFuncPtr(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* baseAddr = inst->src(0);
  SSATmp* offset   = inst->src(1);

  auto dstReg  = m_regs[dst].reg();
  auto baseReg = m_regs[baseAddr].reg();

  assert(offset->isConst());

  m_as.load_reg64_disp_reg64(baseReg,
                           offset->getValInt() + AROFF(m_func),
                           dstReg);
}

static int getNativeTypeSize(Type type) {
  if (type.subtypeOf(Type::Int | Type::Func)) return sz::qword;
  if (type.subtypeOf(Type::Bool))             return sz::byte;
  not_implemented();
}

void CodeGenerator::cgLdRaw(IRInstruction* inst) {
  SSATmp* dest   = inst->dst();
  SSATmp* addr   = inst->src(0);
  SSATmp* offset = inst->src(1);

  assert(!(dest->isConst()));

  Reg64 addrReg = m_regs[addr].reg();
  PhysReg destReg = m_regs[dest].reg();

  if (addr->isConst()) {
    addrReg = m_rScratch;
    emitLoadImm(m_as, addr->getValRawInt(), addrReg);
  }

  if (offset->isConst()) {
    assert(offset->type() == Type::Int);
    int64_t kind = offset->getValInt();
    RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
    int ldSize = slot.size();
    int64_t off = slot.offset();
    if (ldSize == sz::qword) {
      m_as.loadq (addrReg[off], destReg);
    } else if (ldSize == sz::dword) {
      m_as.loadl (addrReg[off], r32(destReg));
    } else {
      assert(ldSize == sz::byte);
      m_as.loadzbl (addrReg[off], r32(destReg));
    }
  } else {
    int ldSize = getNativeTypeSize(dest->type());
    Reg64 offsetReg = r64(m_regs[offset].reg());
    if (ldSize == sz::qword) {
      m_as.loadq (addrReg[offsetReg], destReg);
    } else {
      // Not yet supported by our assembler
      assert(ldSize == sz::byte);
      not_implemented();
    }
  }
}

void CodeGenerator::cgStRaw(IRInstruction* inst) {
  auto baseReg = m_regs[inst->src(0)].reg();
  int64_t kind = inst->src(1)->getValInt();
  SSATmp* value = inst->src(2);

  RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
  assert(value->type().equals(slot.type()));
  int stSize = slot.size();
  int64_t off = slot.offset();
  auto dest = baseReg[off];

  if (value->isConst()) {
    if (stSize == sz::qword) {
      m_as.storeq(value->getValRawInt(), dest);
    } else if (stSize == sz::dword) {
      m_as.storel(value->getValRawInt(), dest);
    } else {
      assert(stSize == sz::byte);
      m_as.storeb(value->getValBool(), dest);
    }
  } else {
    if (stSize == sz::qword) {
      m_as.storeq(r64(m_regs[value].reg()), dest);
    } else if (stSize == sz::dword) {
      m_as.storel(r32(m_regs[value].reg()), dest);
    } else {
      assert(stSize == sz::byte);
      m_as.storeb(rbyte(m_regs[value].reg()), dest);
    }
  }
}

void CodeGenerator::cgLdStaticLocCached(IRInstruction* inst) {
  auto ch = inst->src(0)->getValRawInt();
  auto outReg = m_regs[inst->dst()].reg();

  m_as.loadq (rVmTl[ch], outReg);
  m_as.testq (outReg, outReg);
  emitFwdJcc(m_as, CC_Z, inst->taken());
}

// If label is set and type is not Gen, this method generates a check
// that bails to the label if the loaded typed value doesn't match type.
void CodeGenerator::cgLoadTypedValue(PhysReg base,
                                     int64_t off,
                                     IRInstruction* inst) {
  Block* label = inst->taken();
  Type type   = inst->typeParam();
  SSATmp* dst = inst->dst();

  assert(type == dst->type());
  assert(type.needsReg());
  auto valueDstReg = m_regs[dst].reg(0);
  auto typeDstReg  = m_regs[dst].reg(1);

  if (valueDstReg.isXMM()) {
    // Whole typed value is stored in single XMM reg valueDstReg
    assert(RuntimeOption::EvalHHIRAllocXMMRegs);
    assert(typeDstReg == InvalidReg);
    m_as.movdqa(base[off + TVOFF(m_data)], valueDstReg);
    return;
  }

  if (valueDstReg == InvalidReg && typeDstReg == InvalidReg &&
      (label == nullptr || type == Type::Gen)) {
    // a dead load
    return;
  }
  bool useScratchReg = (base == typeDstReg && valueDstReg != InvalidReg);
  if (useScratchReg) {
    // Save base to m_rScratch, because base will be overwritten.
    m_as.mov_reg64_reg64(base, m_rScratch);
  }

  // Load type if it's not dead
  if (typeDstReg != InvalidReg) {
    emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg);
    if (label) {
      emitTypeCheck(inst->typeParam(), typeDstReg,
                    valueDstReg, inst->taken());
    }
  } else if (label) {
    emitTypeCheck(inst->typeParam(),
                  base[off + TVOFF(m_type)],
                  base[off + TVOFF(m_data)],
                  inst->taken());
  }

  // Load value if it's not dead
  if (valueDstReg == InvalidReg) return;

  if (useScratchReg) {
    m_as.loadq(m_rScratch[off + TVOFF(m_data)], valueDstReg);
  } else {
    m_as.loadq(base[off + TVOFF(m_data)], valueDstReg);
  }
}

void CodeGenerator::cgStoreTypedValue(PhysReg base,
                                      int64_t off,
                                      SSATmp* src) {
  assert(src->type().needsReg());
  auto srcReg0 = m_regs[src].reg(0);
  auto srcReg1 = m_regs[src].reg(1);
  if (srcReg0.isXMM()) {
    // Whole typed value is stored in single XMM reg srcReg0
    assert(RuntimeOption::EvalHHIRAllocXMMRegs);
    assert(srcReg1 == InvalidReg);
    m_as.movdqa(srcReg0, base[off + TVOFF(m_data)]);
    return;
  }
  m_as.storeq(srcReg0, base[off + TVOFF(m_data)]);
  emitStoreTVType(m_as, srcReg1, base[off + TVOFF(m_type)]);
}

void CodeGenerator::cgStore(PhysReg base,
                            int64_t off,
                            SSATmp* src,
                            bool genStoreType) {
  Type type = src->type();
  if (type.needsReg()) {
    cgStoreTypedValue(base, off, src);
    return;
  }
  // store the type
  if (genStoreType) {
    emitStoreTVType(m_as, type.toDataType(), base[off + TVOFF(m_type)]);
  }
  if (type.isNull()) {
    // no need to store a value for null or uninit
    return;
  }
  if (src->isConst()) {
    int64_t val = 0;
    if (type.subtypeOf(Type::Bool | Type::Int | Type::Dbl |
                       Type::Arr | Type::StaticStr | Type::Cls)) {
        val = src->getValBits();
    } else {
      not_reached();
    }
    m_as.storeq(val, base[off + TVOFF(m_data)]);
  } else {
    zeroExtendIfBool(m_as, src, m_regs[src]);
    emitStoreReg(m_as, m_regs[src].reg(), base[off + TVOFF(m_data)]);
  }
}

void CodeGenerator::cgLoad(PhysReg base,
                           int64_t off,
                           IRInstruction* inst) {
  Type type = inst->typeParam();
  if (type.needsReg()) {
    return cgLoadTypedValue(base, off, inst);
  }
  Block* label = inst->taken();
  if (label != NULL) {
    emitTypeCheck(inst->typeParam(),
                  base[off + TVOFF(m_type)],
                  base[off + TVOFF(m_data)],
                  inst->taken());
  }
  if (type.isNull()) return; // these are constants
  auto dstReg = m_regs[inst->dst()].reg();
  // if dstReg == InvalidReg then the value of this load is dead
  if (dstReg == InvalidReg) return;

  if (type == Type::Bool) {
    m_as.load_reg64_disp_reg32(base, off + TVOFF(m_data),  dstReg);
  } else {
    emitLoadReg(m_as, base[off + TVOFF(m_data)],  dstReg);
  }
}

void CodeGenerator::cgLdProp(IRInstruction* inst) {
  cgLoad(m_regs[inst->src(0)].reg(), inst->src(1)->getValInt(), inst);
}

void CodeGenerator::cgLdMem(IRInstruction * inst) {
  cgLoad(m_regs[inst->src(0)].reg(), inst->src(1)->getValInt(), inst);
}

void CodeGenerator::cgLdRef(IRInstruction* inst) {
  cgLoad(m_regs[inst->src(0)].reg(), RefData::tvOffset(), inst);
}

void CodeGenerator::recordSyncPoint(Asm& as,
                                    SyncOptions sync /* = kSyncPoint */) {
  assert(m_state.lastMarker);

  Offset stackOff = m_state.lastMarker->stackOff;
  switch (sync) {
  case kSyncPointAdjustOne:
    stackOff -= 1;
    break;
  case kSyncPoint:
    break;
  case kNoSyncPoint:
    assert(0);
  }

  Offset pcOff = m_state.lastMarker->bcOff - m_state.lastMarker->func->base();

  FTRACE(5, "IR recordSyncPoint: {} {} {}\n", as.code.frontier, pcOff,
         stackOff);
  m_tx64->recordSyncPoint(as, pcOff, stackOff);
}

void CodeGenerator::cgLdAddr(IRInstruction* inst) {
  auto base = m_regs[inst->src(0)].reg();
  int64_t offset = inst->src(1)->getValInt();
  m_as.lea (base[offset], m_regs[inst->dst()].reg());
}

void CodeGenerator::cgLdLoc(IRInstruction* inst) {
  cgLoad(m_regs[inst->src(0)].reg(),
         localOffset(inst->extra<LdLoc>()->locId),
         inst);
}

void CodeGenerator::cgLdLocAddr(IRInstruction* inst) {
  auto const fpReg  = m_regs[inst->src(0)].reg();
  auto const offset = localOffset(inst->extra<LdLocAddr>()->locId);
  if (m_regs[inst->dst()].hasReg()) {
    m_as.lea(fpReg[offset], m_regs[inst->dst()].reg());
  }
}

void CodeGenerator::cgLdStackAddr(IRInstruction* inst) {
  auto const base   = m_regs[inst->src(0)].reg();
  auto const offset = cellsToBytes(inst->extra<LdStackAddr>()->offset);
  m_as.lea (base[offset], m_regs[inst->dst()].reg());
}

void CodeGenerator::cgLdStack(IRInstruction* inst) {
  assert(inst->taken() == nullptr);
  cgLoad(m_regs[inst->src(0)].reg(),
         cellsToBytes(inst->extra<LdStack>()->offset),
         inst);
}

void CodeGenerator::cgGuardStk(IRInstruction* inst) {
  auto const rSP = m_regs[inst->src(0)].reg();
  auto const baseOff = cellsToBytes(inst->extra<GuardStk>()->offset);
  emitTypeGuard(inst->typeParam(),
                rSP[baseOff + TVOFF(m_type)],
                rSP[baseOff + TVOFF(m_data)]);
}

void CodeGenerator::cgCheckStk(IRInstruction* inst) {
  auto const rbase = m_regs[inst->src(0)].reg();
  auto const baseOff = cellsToBytes(inst->extra<CheckStk>()->offset);
  emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
                rbase[baseOff + TVOFF(m_data)], inst->taken());
}

void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
  auto const rFP = m_regs[inst->src(0)].reg();
  auto const baseOff = localOffset(inst->extra<GuardLoc>()->locId);
  emitTypeGuard(inst->typeParam(),
                rFP[baseOff + TVOFF(m_type)],
                rFP[baseOff + TVOFF(m_data)]);
}

void CodeGenerator::cgCheckLoc(IRInstruction* inst) {
  auto const rbase = m_regs[inst->src(0)].reg();
  auto const baseOff = localOffset(inst->extra<CheckLoc>()->locId);
  emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
                rbase[baseOff + TVOFF(m_data)], inst->taken());
}

template<class Loc>
void CodeGenerator::emitSideExitGuard(Type type,
                                      Loc typeSrc,
                                      Loc dataSrc,
                                      Offset taken) {
  emitTypeTest(type, typeSrc, dataSrc,
    [&](ConditionCode cc) {
      auto const sk = SrcKey(curFunc(), taken);
      m_tx64->emitBindJcc(m_as, ccNegate(cc), sk, REQ_BIND_SIDE_EXIT);
  });
}

void CodeGenerator::cgSideExitGuardLoc(IRInstruction* inst) {
  auto const fp    = m_regs[inst->src(0)].reg();
  auto const extra = inst->extra<SideExitGuardLoc>();
  emitSideExitGuard(inst->typeParam(),
                    fp[localOffset(extra->checkedSlot) + TVOFF(m_type)],
                    fp[localOffset(extra->checkedSlot) + TVOFF(m_data)],
                    extra->taken);
}

void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) {
  auto const sp    = m_regs[inst->src(0)].reg();
  auto const extra = inst->extra<SideExitGuardStk>();
  emitSideExitGuard(inst->typeParam(),
                    sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)],
                    sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_data)],
                    extra->taken);
}

void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) {
  assert(inst->dst()->type() == Type::PtrToCell);
  assert(m_regs[inst->dst()].reg() == rsp);
}

void CodeGenerator::cgCheckType(IRInstruction* inst) {
  auto const src   = inst->src(0);
  auto const t     = inst->typeParam();
  auto const rData = m_regs[src].reg(0);
  auto const rType = m_regs[src].reg(1);

  auto doJcc = [&](ConditionCode cc) {
    emitFwdJcc(ccNegate(cc), inst->taken());
  };

  if (t.equals(Type::Nullptr)) {
    if (!src->type().equals(Type::Nullptr | Type::CountedStr)) {
      CG_PUNT(CheckType-Nullptr-UnsupportedType);
    }
    m_as.testq (rData, rData);
    doJcc(CC_E);
  } else {
    emitTypeTest(inst->typeParam(), rType, rData, doJcc);
  }

  auto const dstReg = m_regs[inst->dst()].reg();
  if (dstReg != InvalidReg) {
    emitMovRegReg(m_as, rData, dstReg);
  }
}

void CodeGenerator::cgCheckTypeMem(IRInstruction* inst) {
  auto const reg = m_regs[inst->src(0)].reg();
  emitTypeCheck(inst->typeParam(), reg[TVOFF(m_type)],
                reg[TVOFF(m_data)], inst->taken());
}

void CodeGenerator::cgGuardRefs(IRInstruction* inst) {
  assert(inst->numSrcs() == 5);

  SSATmp* funcPtrTmp = inst->src(0);
  SSATmp* nParamsTmp = inst->src(1);
  SSATmp* firstBitNumTmp = inst->src(2);
  SSATmp* mask64Tmp  = inst->src(3);
  SSATmp* vals64Tmp  = inst->src(4);

  // Get values in place
  assert(funcPtrTmp->type() == Type::Func);
  auto funcPtrReg = m_regs[funcPtrTmp].reg();
  assert(funcPtrReg != InvalidReg);

  assert(nParamsTmp->type() == Type::Int);
  auto nParamsReg = m_regs[nParamsTmp].reg();
  assert(nParamsReg != InvalidReg || nParamsTmp->isConst());

  assert(firstBitNumTmp->isConst() && firstBitNumTmp->type() == Type::Int);
  uint32_t firstBitNum = (uint32_t)(firstBitNumTmp->getValInt());

  assert(mask64Tmp->type() == Type::Int);
  assert(mask64Tmp->isConst());
  auto mask64Reg = m_regs[mask64Tmp].reg();
  assert(mask64Reg != InvalidReg || mask64Tmp->inst()->op() != LdConst);
  uint64_t mask64 = mask64Tmp->getValInt();
  assert(mask64);

  assert(vals64Tmp->type() == Type::Int);
  assert(vals64Tmp->isConst());
  auto vals64Reg = m_regs[vals64Tmp].reg();
  assert(vals64Reg != InvalidReg || vals64Tmp->inst()->op() != LdConst);
  uint64_t vals64 = vals64Tmp->getValInt();
  assert((vals64 & mask64) == vals64);

  auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
  auto const destSR = m_tx64->getSrcRec(destSK);

  auto thenBody = [&] {
    auto bitsOff = sizeof(uint64_t) * (firstBitNum / 64);
    auto cond = CC_NE;
    auto bitsPtrReg = m_rScratch;

    if (firstBitNum == 0) {
      bitsOff = Func::refBitValOff();
      bitsPtrReg = funcPtrReg;
    } else {
      m_as.loadq(funcPtrReg[Func::sharedOff()], bitsPtrReg);
      bitsOff -= sizeof(uint64_t);
    }

    if (vals64 == 0 || (mask64 & (mask64 - 1)) == 0) {
      // If vals64 is zero, or we're testing a single
      // bit, we can get away with a single test,
      // rather than mask-and-compare
      if (mask64Reg != InvalidReg) {
        m_as.testq  (mask64Reg, bitsPtrReg[bitsOff]);
      } else {
        if (mask64 < 256) {
          m_as.testb((int8_t)mask64, bitsPtrReg[bitsOff]);
        } else {
          m_as.testl((int32_t)mask64, bitsPtrReg[bitsOff]);
        }
      }
      if (vals64) cond = CC_E;
    } else {
      auto bitsValReg = m_rScratch;
      m_as.  loadq  (bitsPtrReg[bitsOff], bitsValReg);
      if (debug) bitsPtrReg = InvalidReg;

      //     bitsValReg <- bitsValReg & mask64
      if (mask64Reg != InvalidReg) {
        m_as.  andq   (mask64Reg, bitsValReg);
      } else if (mask64 < 256) {
        m_as.  andb   ((int8_t)mask64, rbyte(bitsValReg));
      } else {
        m_as.  andl   ((int32_t)mask64, r32(bitsValReg));
      }

      //   If bitsValReg != vals64, then goto Exit
      if (vals64Reg != InvalidReg) {
        m_as.  cmpq   (vals64Reg, bitsValReg);
      } else if (mask64 < 256) {
        assert(vals64 < 256);
        m_as.  cmpb   ((int8_t)vals64, rbyte(bitsValReg));
      } else {
        m_as.  cmpl   ((int32_t)vals64, r32(bitsValReg));
      }
    }
    m_tx64->emitFallbackCondJmp(m_as, *destSR, cond);
  };

  if (firstBitNum == 0) {
    assert(nParamsReg == InvalidReg);
    // This is the first 64 bits. No need to check
    // nParams.
    thenBody();
  } else {
    assert(nParamsReg != InvalidReg);
    // Check number of args...
    m_as.    cmpq   (firstBitNum, nParamsReg);

    if (vals64 != 0 && vals64 != mask64) {
      // If we're beyond nParams, then either all params
      // are refs, or all params are non-refs, so if vals64
      // isn't 0 and isnt mask64, there's no possibility of
      // a match
      m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_LE);
      thenBody();
    } else {
      ifThenElse(CC_NLE, thenBody, /* else */ [&] {
          //   If not special builtin...
          m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
          m_tx64->emitFallbackCondJmp(m_as, *destSR, vals64 ? CC_Z : CC_NZ);
        });
    }
  }
}

void CodeGenerator::cgLdPropAddr(IRInstruction* inst) {
  SSATmp*   dst   = inst->dst();
  SSATmp*   obj   = inst->src(0);
  SSATmp*   prop  = inst->src(1);

  assert(prop->isConst() && prop->type() == Type::Int);

  auto dstReg = m_regs[dst].reg();
  auto objReg = m_regs[obj].reg();

  assert(objReg != InvalidReg);
  assert(dstReg != InvalidReg);

  int64_t offset = prop->getValInt();
  m_as.lea_reg64_disp_reg64(objReg, offset, dstReg);
}

void CodeGenerator::cgLdClsMethod(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* cls   = inst->src(0);
  SSATmp* mSlot = inst->src(1);

  assert(cls->type() == Type::Cls);
  assert(mSlot->isConst() && mSlot->type() == Type::Int);
  uint64_t mSlotInt64 = mSlot->getValRawInt();
  // We're going to multiply mSlotVal by sizeof(Func*) and use
  // it as a 32-bit offset (methOff) below.
  if (mSlotInt64 > (std::numeric_limits<uint32_t>::max() / sizeof(Func*))) {
    CG_PUNT(cgLdClsMethod_large_offset);
  }
  int32_t mSlotVal = (uint32_t) mSlotInt64;

  Reg64 dstReg = m_regs[dst].reg();
  assert(dstReg != InvalidReg);

  Reg64 clsReg = m_regs[cls].reg();
  if (clsReg == InvalidReg) {
    CG_PUNT(LdClsMethod);
  }

  Offset vecOff  = Class::getMethodsOffset() + Class::MethodMap::vecOff();
  int32_t  methOff = mSlotVal * sizeof(Func*);
  m_as.loadq(clsReg[vecOff],  dstReg);
  m_as.loadq(dstReg[methOff], dstReg);
}

void CodeGenerator::cgLdClsMethodCache(IRInstruction* inst) {
  SSATmp* dst        = inst->dst();
  SSATmp* className  = inst->src(0);
  SSATmp* methodName = inst->src(1);
  SSATmp* baseClass  = inst->src(2);
  Block*  label      = inst->taken();

  // Stats::emitInc(a, Stats::TgtCache_StaticMethodHit);
  const StringData*  cls    = className->getValStr();
  const StringData*  method = methodName->getValStr();
  auto const ne             = baseClass->getValNamedEntity();
  TargetCache::CacheHandle ch =
    TargetCache::StaticMethodCache::alloc(cls,
                                          method,
                                          getContextName(curClass()));
  auto funcDestReg  = m_regs[dst].reg(0);
  auto classDestReg = m_regs[dst].reg(1);
  auto offsetof_func = offsetof(TargetCache::StaticMethodCache, m_func);
  auto offsetof_cls  = offsetof(TargetCache::StaticMethodCache, m_cls);

  assert(funcDestReg != InvalidReg && classDestReg != InvalidReg);
  // Attempt to retrieve the func* and class* from cache
  m_as.loadq(rVmTl[ch + offsetof_func], funcDestReg);
  m_as.loadq(rVmTl[ch + offsetof_cls], classDestReg);
  m_as.testq(funcDestReg, funcDestReg);
  // May have retrieved a NULL from the cache
  // handle case where method is not entered in the cache
  unlikelyIfBlock(CC_E, [&] (Asm& a) {
    if (false) { // typecheck
      const UNUSED Func* f = StaticMethodCache::lookupIR(ch, ne, cls, method);
    }
    // can raise an error if class is undefined
    cgCallHelper(a,
                 (TCA)StaticMethodCache::lookupIR,
                 funcDestReg,
                 kSyncPoint,
                 ArgGroup(m_regs).imm(ch)      // Handle ch
                           .immPtr(ne)         // NamedEntity* np.second
                           .immPtr(cls)        // className
                           .immPtr(method)     // methodName
    );
    // recordInstrCall is done in cgCallHelper
    a.testq(funcDestReg, funcDestReg);
    a.loadq(rVmTl[ch + offsetof_cls], classDestReg);
    // if StaticMethodCache::lookupIR() returned NULL, jmp to label
    emitFwdJcc(a, CC_Z, label);
  });
}

/**
 * Helper to emit getting the value for ActRec's m_this/m_cls slot
 * from a This pointer depending on whether the callee method is
 * static or not.
 */
void CodeGenerator::emitGetCtxFwdCallWithThis(PhysReg ctxReg,
                                              bool    staticCallee) {
  if (staticCallee) {
    // Load (this->m_cls | 0x1) into ctxReg.
    m_as.loadq(ctxReg[ObjectData::getVMClassOffset()], ctxReg);
    m_as.orq(1, ctxReg);
  } else {
    // Just incref $this.
    emitIncRef(m_as, ctxReg);
  }
}

/**
 * This method is similar to emitGetCtxFwdCallWithThis above, but
 * whether or not the callee is a static method is unknown at JIT
 * time, and that is determined dynamically by looking up into the
 * StaticMethodFCache.
 */
void CodeGenerator::emitGetCtxFwdCallWithThisDyn(PhysReg      destCtxReg,
                                                 PhysReg      thisReg,
                                                 CacheHandle& ch) {
  Label NonStaticCall, End;

  // thisReg is holding $this. Should we pass it to the callee?
  m_as.cmpl(1, rVmTl[ch + offsetof(StaticMethodFCache, m_static)]);
  m_as.jcc8(CC_NE, NonStaticCall);
  // If calling a static method...
  {
    // Load (this->m_cls | 0x1) into destCtxReg
    m_as.loadq(thisReg[ObjectData::getVMClassOffset()], destCtxReg);
    m_as.orq(1, destCtxReg);
    m_as.jmp8(End);
  }
  // Else: calling non-static method
  {
    asm_label(m_as, NonStaticCall);
    emitMovRegReg(m_as, thisReg, destCtxReg);
    emitIncRef(m_as, destCtxReg);
  }
  asm_label(m_as, End);
}

void CodeGenerator::cgGetCtxFwdCall(IRInstruction* inst) {
  PhysReg destCtxReg = m_regs[inst->dst()].reg(0);
  SSATmp*  srcCtxTmp = inst->src(0);
  const Func* callee = inst->src(1)->getValFunc();
  bool      withThis = srcCtxTmp->isA(Type::Obj);

  // Eagerly move src into the dest reg
  emitMovRegReg(m_as, m_regs[srcCtxTmp].reg(0), destCtxReg);

  Label End;
  // If we don't know whether we have a This, we need to check dynamically
  if (!withThis) {
    m_as.testb(1, rbyte(destCtxReg));
    m_as.jcc8(CC_NZ, End);
  }

  // If we have a This pointer in destCtxReg, then select either This
  // or its Class based on whether callee is static or not
  emitGetCtxFwdCallWithThis(destCtxReg, (callee->attrs() & AttrStatic));

  asm_label(m_as, End);
}

void CodeGenerator::cgLdClsMethodFCache(IRInstruction* inst) {
  PhysReg         funcDestReg = m_regs[inst->dst()].reg(0);
  PhysReg          destCtxReg = m_regs[inst->dst()].reg(1);
  const Class*            cls = inst->src(0)->getValClass();
  const StringData*  methName = inst->src(1)->getValStr();
  SSATmp*           srcCtxTmp = inst->src(2);
  PhysReg           srcCtxReg = m_regs[srcCtxTmp].reg(0);
  Block*            exitLabel = inst->taken();
  const StringData*   clsName = cls->name();
  CacheHandle              ch = StaticMethodFCache::alloc(clsName, methName,
                                              getContextName(curClass()));

  assert(funcDestReg != InvalidReg && destCtxReg != InvalidReg);
  emitMovRegReg(m_as, srcCtxReg, destCtxReg);
  m_as.loadq(rVmTl[ch], funcDestReg);
  m_as.testq(funcDestReg, funcDestReg);

  Label End;

  // Handle case where method is not entered in the cache
  unlikelyIfBlock(CC_E, [&] (Asm& a) {
    const Func* (*lookup)(CacheHandle, const Class*, const StringData*) =
      StaticMethodFCache::lookupIR;
    // preserve destCtxReg across the call since it wouldn't be otherwise
    RegSet toSave = m_state.liveRegs[inst] | RegSet(destCtxReg);
    cgCallHelper(a, Transl::CppCall((TCA)lookup),
                 funcDestReg, InvalidReg,
                 kSyncPoint,
                 ArgGroup(m_regs).imm(ch)
                           .immPtr(cls)
                           .immPtr(methName),
                 toSave);
    // If entry found in target cache, jump back to m_as.
    // Otherwise, bail to exit label
    a.testq(funcDestReg, funcDestReg);
    emitFwdJcc(a, CC_Z, exitLabel);
  });

  auto t = srcCtxTmp->type();
  assert(!t.equals(Type::Cls));
  if (t.equals(Type::Cctx)) {
    return; // done: destCtxReg already has srcCtxReg
  } else if (t == Type::Obj) {
    // unconditionally run code produced by emitGetCtxFwdCallWithThisDyn below
    // break
  } else if (t == Type::Ctx) {
    // dynamically check if we have a This pointer and
    // call emitGetCtxFwdCallWithThisDyn below
    m_as.testb(1, rbyte(destCtxReg));
    m_as.jcc8(CC_NZ, End);
  } else {
    not_reached();
  }

  // If we have a 'this' pointer ...
  emitGetCtxFwdCallWithThisDyn(destCtxReg, destCtxReg, ch);

  asm_label(m_as, End);
}

void CodeGenerator::cgLdClsPropAddrCached(IRInstruction* inst) {
  using namespace Transl::TargetCache;
  SSATmp* dst      = inst->dst();
  SSATmp* cls      = inst->src(0);
  SSATmp* propName = inst->src(1);
  SSATmp* clsName  = inst->src(2);
  SSATmp* cxt      = inst->src(3);
  Block* target    = inst->taken();

  const StringData* propNameString = propName->getValStr();
  const StringData* clsNameString  = clsName->getValStr();

  string sds(Util::toLower(clsNameString->data()) + ":" +
             string(propNameString->data(), propNameString->size()));
  StackStringData sd(sds.c_str(), sds.size(), AttachLiteral);
  CacheHandle ch = SPropCache::alloc(&sd);

  auto dstReg = m_regs[dst].reg();
  // Cls is live in the slow path call to lookupIR, so we have to be
  // careful not to clobber it before the branch to slow path. So
  // use the scratch register as a temporary destination if cls is
  // assigned the same register as the dst register.
  auto tmpReg = dstReg;
  if (dstReg == InvalidReg || dstReg == m_regs[cls].reg()) {
    tmpReg = PhysReg(m_rScratch);
  }

  // Could be optimized to cmp against zero when !label && dstReg == InvalidReg
  m_as.loadq(rVmTl[ch], tmpReg);
  m_as.testq(tmpReg, tmpReg);
  unlikelyIfBlock(CC_E, [&] (Asm& a) {
    cgCallHelper(a,
                 target ? (TCA)SPropCache::lookupIR<false>
                        : (TCA)SPropCache::lookupIR<true>, // raise on error
                 tmpReg,
                 kSyncPoint, // could re-enter to initialize properties
                 ArgGroup(m_regs).imm(ch).ssa(cls).ssa(propName).ssa(cxt));
    if (target) {
      a.testq(tmpReg, tmpReg);
      emitFwdJcc(a, CC_Z, target);
    }
  });
  if (dstReg != InvalidReg) {
    emitMovRegReg(m_as, tmpReg, dstReg);
  }
}

void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) {
  SSATmp* dst    = inst->dst();
  SSATmp* cls    = inst->src(0);
  SSATmp* prop   = inst->src(1);
  SSATmp* ctx    = inst->src(2);
  Block*  target = inst->taken();
  // If our label is a catch trace we pretend we don't have one, to
  // avoid emitting a jmp to it or calling the wrong helper.
  if (target && target->trace()->isCatch()) target = nullptr;

  auto dstReg = m_regs[dst].reg();
  if (dstReg == InvalidReg && target) {
    // result is unused but this instruction was not eliminated
    // because its essential
    dstReg = m_rScratch;
  }
  cgCallHelper(m_as,
               target ? (TCA)SPropCache::lookupSProp<false>
                      : (TCA)SPropCache::lookupSProp<true>, // raise on error
               dstReg,
               kSyncPoint, // could re-enter to initialize properties
               ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(ctx));
  if (target) {
    m_as.testq(dstReg, dstReg);
    emitFwdJcc(m_as, CC_Z, target);
  }
}

TargetCache::CacheHandle CodeGenerator::cgLdClsCachedCommon(
  IRInstruction* inst) {
  SSATmp* dst = inst->dst();
  const StringData* className = inst->src(0)->getValStr();
  auto ch = TargetCache::allocKnownClass(className);
  auto dstReg = m_regs[dst].reg();
  if (dstReg == InvalidReg) {
    m_as. cmpq   (0, rVmTl[ch]);
  } else {
    m_as.  loadq  (rVmTl[ch], dstReg);
    m_as.  testq  (dstReg, dstReg);
  }

  return ch;
}

void CodeGenerator::cgLdClsCached(IRInstruction* inst) {
  auto ch = cgLdClsCachedCommon(inst);
  unlikelyIfBlock(CC_E, [&] (Asm& a) {
    // Passing only two arguments to lookupKnownClass, since the
    // third is ignored in the checkOnly==false case.
    cgCallHelper(a,
                 (TCA)TargetCache::lookupKnownClass<false>,
                 inst->dst(),
                 kSyncPoint,
                 ArgGroup(m_regs).addr(rVmTl, intptr_t(ch)).ssas(inst, 0));
  });
}

void CodeGenerator::cgLdClsCachedSafe(IRInstruction* inst) {
  cgLdClsCachedCommon(inst);
  if (Block* taken = inst->taken()) {
    emitFwdJcc(CC_Z, taken);
  }
}

void CodeGenerator::cgLdCls(IRInstruction* inst) {
  SSATmp* dst       = inst->dst();
  SSATmp* className = inst->src(0);

  CacheHandle ch = ClassCache::alloc();
  cgCallHelper(m_as, (TCA)ClassCache::lookup, dst, kSyncPoint,
               ArgGroup(m_regs).imm(ch).ssa(className));
}

static StringData* fullConstName(const StringData* cls,
                                 const StringData* cnsName) {
  return StringData::GetStaticString(
    Util::toLower(cls->data()) + "::" + cnsName->data()
  );
}

void CodeGenerator::cgLdClsCns(IRInstruction* inst) {
  auto const extra    = inst->extra<LdClsCns>();
  auto const fullName = fullConstName(extra->clsName, extra->cnsName);
  auto const ch       = TargetCache::allocClassConstant(fullName);
  cgLoad(rVmTl, ch, inst);
}

void CodeGenerator::cgLookupClsCns(IRInstruction* inst) {
  auto const extra    = inst->extra<LookupClsCns>();
  auto const fullName = fullConstName(extra->clsName, extra->cnsName);
  auto const ch       = TargetCache::allocClassConstant(fullName);
  cgCallHelper(
    m_as,
    TCA(TargetCache::lookupClassConstantTv),
    inst->dst(),
    kSyncPoint,
    ArgGroup(m_regs)
      .addr(rVmTl, ch)
      .immPtr(Unit::GetNamedEntity(extra->clsName))
      .immPtr(extra->clsName)
      .immPtr(extra->cnsName),
    DestType::TV
  );
}

void CodeGenerator::cgLdCns(IRInstruction* inst) {
  const StringData* cnsName = inst->src(0)->getValStr();

  TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);
  // Has an unlikely branch to a LookupCns
  cgLoad(rVmTl, ch, inst);
}

static TypedValue lookupCnsHelper(const TypedValue* tv, StringData* nm) {
  assert(tv->m_type == KindOfUninit);
  TypedValue *cns = nullptr;
  TypedValue c1;
  if (UNLIKELY(tv->m_data.pref != nullptr)) {
    ClassInfo::ConstantInfo* ci =
      (ClassInfo::ConstantInfo*)(void*)tv->m_data.pref;
    cns = const_cast<Variant&>(ci->getDeferredValue()).asTypedValue();
    tvReadCell(cns, &c1);
  } else {
    if (UNLIKELY(TargetCache::s_constants != nullptr)) {
      cns = TargetCache::s_constants->HphpArray::nvGet(nm);
    }
    if (!cns) {
      cns = Unit::loadCns(const_cast<StringData*>(nm));
    }
    if (UNLIKELY(!cns)) {
      raise_notice(Strings::UNDEFINED_CONSTANT, nm->data(), nm->data());
      c1.m_data.pstr = const_cast<StringData*>(nm);
      c1.m_type = KindOfStaticString;
    } else {
      c1.m_type = cns->m_type;
      c1.m_data = cns->m_data;
    }
  }
  return c1;
}

void CodeGenerator::cgLookupCns(IRInstruction* inst) {
  SSATmp* cnsNameTmp = inst->src(0);

  assert(inst->typeParam() == Type::Cell);
  assert(cnsNameTmp->isConst() && cnsNameTmp->type() == Type::StaticStr);

  const StringData* cnsName = cnsNameTmp->getValStr();
  TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);

  ArgGroup args(m_regs);
  args.addr(rVmTl, ch)
      .immPtr(cnsName);

  cgCallHelper(m_as, TCA(lookupCnsHelper),
               inst->dst(), kSyncPoint, args, DestType::TV);
}

HOT_FUNC_VM
static inline int64_t ak_exist_string_helper(StringData* key, ArrayData* arr) {
  int64_t n;
  if (key->isStrictlyInteger(n)) {
    return arr->exists(n);
  }
  return arr->exists(StrNR(key));
}

HOT_FUNC_VM
static int64_t ak_exist_string(StringData* key, ArrayData* arr) {
  int64_t res = ak_exist_string_helper(key, arr);
  return res;
}

HOT_FUNC_VM
static int64_t ak_exist_int(int64_t key, ArrayData* arr) {
  bool res = arr->exists(key);
  return res;
}

HOT_FUNC_VM
static int64_t ak_exist_string_obj(StringData* key, ObjectData* obj) {
  if (obj->isCollection()) {
    return collectionOffsetContains(obj, key);
  }
  CArrRef arr = obj->o_toArray();
  int64_t res = ak_exist_string_helper(key, arr.get());
  return res;
}

HOT_FUNC_VM
static int64_t ak_exist_int_obj(int64_t key, ObjectData* obj) {
  if (obj->isCollection()) {
    return collectionOffsetContains(obj, key);
  }
  CArrRef arr = obj->o_toArray();
  bool res = arr.get()->exists(key);
  return res;
}

void CodeGenerator::cgAKExists(IRInstruction* inst) {
  SSATmp* arr = inst->src(0);
  SSATmp* key = inst->src(1);

  if (key->type().isNull()) {
    if (arr->isA(Type::Arr)) {
      cgCallHelper(m_as,
                   (TCA)ak_exist_string,
                   inst->dst(),
                   kNoSyncPoint,
                   ArgGroup(m_regs).immPtr(empty_string.get()).ssa(arr));
    } else {
      m_as.mov_imm64_reg(0, m_regs[inst->dst()].reg());
    }
    return;
  }

  TCA helper_func =
    arr->isA(Type::Obj)
    ? (key->isA(Type::Int) ? (TCA)ak_exist_int_obj : (TCA)ak_exist_string_obj)
    : (key->isA(Type::Int) ? (TCA)ak_exist_int : (TCA)ak_exist_string);

  cgCallHelper(m_as,
               helper_func,
               inst->dst(),
               kNoSyncPoint,
               ArgGroup(m_regs).ssa(key).ssa(arr));
}

HOT_FUNC_VM static TypedValue* ldGblAddrHelper(StringData* name) {
  return g_vmContext->m_globalVarEnv->lookup(name);
}

HOT_FUNC_VM static TypedValue* ldGblAddrDefHelper(StringData* name) {
  TypedValue* r = g_vmContext->m_globalVarEnv->lookupAdd(name);
  decRefStr(name);
  return r;
}

void CodeGenerator::cgLdGblAddr(IRInstruction* inst) {
  auto dstReg = m_regs[inst->dst()].reg();
  cgCallHelper(m_as, (TCA)ldGblAddrHelper, dstReg, kNoSyncPoint,
               ArgGroup(m_regs).ssa(inst->src(0)));
  m_as.testq(dstReg, dstReg);
  emitFwdJcc(CC_Z, inst->taken());
}

void CodeGenerator::cgLdGblAddrDef(IRInstruction* inst) {
  cgCallHelper(m_as, (TCA)ldGblAddrDefHelper, inst->dst(), kNoSyncPoint,
               ArgGroup(m_regs).ssa(inst->src(0)));
}

void CodeGenerator::emitTestZero(SSATmp* src) {
  auto& a = m_as;
  auto reg = m_regs[src].reg();

  /*
   * If src is const, normally a earlier optimization pass should have
   * converted the thing testing this condition into something
   * unconditional.  So rather than supporting constants efficiently
   * here, we just materialize the value into a register.
   */
  if (reg == InvalidReg) {
    reg = m_rScratch;
    a.    movq   (src->getValBits(), reg);
  }

  if (src->isA(Type::Bool)) {
    a.    testb  (rbyte(reg), rbyte(reg));
  } else {
    a.    testq  (reg, reg);
  }
}

void CodeGenerator::cgJmpZero(IRInstruction* inst) {
  emitTestZero(inst->src(0));
  emitFwdJcc(CC_Z, inst->taken());
}

void CodeGenerator::cgJmpNZero(IRInstruction* inst) {
  emitTestZero(inst->src(0));
  emitFwdJcc(CC_NZ, inst->taken());
}

void CodeGenerator::cgReqBindJmpZero(IRInstruction* inst) {
  // TODO(#2404427): prepareForTestAndSmash?
  emitTestZero(inst->src(0));
  emitReqBindJcc(CC_Z, inst->extra<ReqBindJmpZero>());
}

void CodeGenerator::cgReqBindJmpNZero(IRInstruction* inst) {
  // TODO(#2404427): prepareForTestAndSmash?
  emitTestZero(inst->src(0));
  emitReqBindJcc(CC_NZ, inst->extra<ReqBindJmpNZero>());
}

void CodeGenerator::cgJmp_(IRInstruction* inst) {
  Block* target = inst->taken();
  if (unsigned n = inst->numSrcs()) {
    // Parallel-copy sources to the label's destination registers.
    // TODO: t2040286: this only works if all destinations fit in registers.
    auto srcs = inst->srcs();
    auto dsts = target->front()->dsts();
    ArgGroup args(m_regs);
    for (unsigned i = 0, j = 0; i < n; i++) {
      assert(srcs[i]->type().subtypeOf(dsts[i].type()));
      auto dst = &dsts[i];
      auto src = srcs[i];
      // Currently, full XMM registers cannot be assigned to SSATmps
      // passed from to Jmp_ to DefLabel. If this changes, it'll require
      // teaching shuffleArgs() how to handle full XMM values.
      assert(!m_regs[src].isFullXMM() && !m_regs[dst].isFullXMM());
      if (m_regs[dst].reg(0) == InvalidReg) continue; // dst is unused.
      // first dst register
      args.ssa(src);
      args[j++].setDstReg(m_regs[dst].reg(0));
      // second dst register, if any
      if (dst->numNeededRegs() == 2) {
        if (src->numNeededRegs() < 2) {
          // src has known data type, but dst doesn't - pass immediate type
          assert(src->type().isKnownDataType());
          args.imm(src->type().toDataType());
        } else {
          // pass src's second register
          assert(m_regs[src].reg(1) != InvalidReg);
          args.reg(m_regs[src].reg(1));
        }
        args[j++].setDstReg(m_regs[dst].reg(1));
      }
    }
    assert(args.numStackArgs() == 0 &&
           "Jmp_ doesn't support passing arguments on the stack yet.");
    shuffleArgs(m_as, args);
  }
  if (!m_state.noTerminalJmp_) {
    emitFwdJmp(m_as, target, m_state);
  }
}

void CodeGenerator::cgJmpIndirect(IRInstruction* inst) {
  m_as.jmp(m_regs[inst->src(0)].reg());
}

void CodeGenerator::cgCheckInit(IRInstruction* inst) {
  Block* label = inst->taken();
  assert(label);
  SSATmp* src = inst->src(0);

  if (src->type().isInit()) return;

  auto typeReg = m_regs[src].reg(1);
  assert(typeReg != InvalidReg);

  static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0");
  m_as.testb (rbyte(typeReg), rbyte(typeReg));
  emitFwdJcc(CC_Z, label);
}

void CodeGenerator::cgCheckInitMem(IRInstruction* inst) {
  Block* label = inst->taken();
  assert(label);
  SSATmp* base = inst->src(0);
  int64_t offset = inst->src(1)->getValInt();
  Type t = base->type().deref();
  if (t.isInit()) return;
  auto basereg = m_regs[base].reg();
  emitCmpTVType(m_as, KindOfUninit, basereg[offset + TVOFF(m_type)]);
  emitFwdJcc(CC_Z, label);
}

void CodeGenerator::cgExitWhenSurprised(IRInstruction* inst) {
  Block* label = inst->taken();
  m_tx64->emitTestSurpriseFlags(m_as);
  emitFwdJcc(CC_NZ, label);
}

void CodeGenerator::cgExitOnVarEnv(IRInstruction* inst) {
  SSATmp* fp    = inst->src(0);
  Block*  label = inst->taken();

  assert(!(fp->isConst()));

  auto fpReg = m_regs[fp].reg();
  m_as.    cmpq   (0, fpReg[AROFF(m_varEnv)]);
  emitFwdJcc(CC_NE, label);
}

void CodeGenerator::cgReleaseVVOrExit(IRInstruction* inst) {
  auto* const label = inst->taken();
  auto const rFp = m_regs[inst->src(0)].reg();

  m_as.    cmpq   (0, rFp[AROFF(m_varEnv)]);
  unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
    a.    testl  (ActRec::kExtraArgsBit, rFp[AROFF(m_varEnv)]);
    emitFwdJcc(a, CC_Z, label);
    cgCallHelper(
      a,
      TCA(static_cast<void (*)(ActRec*)>(ExtraArgs::deallocate)),
      nullptr,
      kSyncPoint,
      ArgGroup(m_regs).reg(rFp),
      DestType::None
    );
  });
}

void CodeGenerator::cgBoxPtr(IRInstruction* inst) {
  SSATmp* dst  = inst->dst();
  SSATmp* addr = inst->src(0);
  auto base    = m_regs[addr].reg();
  auto dstReg  = m_regs[dst].reg();
  emitMovRegReg(m_as, base, dstReg);
  emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)],
               base[TVOFF(m_data)],
    [&](ConditionCode cc) {
      ifThen(m_as, ccNegate(cc), [&] {
        cgCallHelper(m_as, (TCA)tvBox, dstReg, kNoSyncPoint,
                     ArgGroup(m_regs).ssa(addr));
      });
    });
}

void CodeGenerator::cgDefCns(IRInstruction* inst) {
  UNUSED SSATmp* dst     = inst->dst();
  UNUSED SSATmp* cnsName = inst->src(0);
  UNUSED SSATmp* val     = inst->src(1);
  using namespace TargetCache;
  CG_PUNT(DefCns);
}

// TODO: Kill this #2031980
static StringData* concat_value(TypedValue tv1, TypedValue tv2) {
  return concat_tv(tv1.m_type, tv1.m_data.num, tv2.m_type, tv2.m_data.num);
}

void CodeGenerator::cgConcat(IRInstruction* inst) {
  SSATmp* dst   = inst->dst();
  SSATmp* tl    = inst->src(0);
  SSATmp* tr    = inst->src(1);

  Type lType = tl->type();
  Type rType = tr->type();
  // We have specialized helpers for concatenating two strings, a
  // string and an int, and an int and a string.
  void* fptr = nullptr;
  if (lType.isString() && rType.isString()) {
    fptr = (void*)concat_ss;
  } else if (lType.isString() && rType == Type::Int) {
    fptr = (void*)concat_si;
  } else if (lType == Type::Int && rType.isString()) {
    fptr = (void*)concat_is;
  }
  if (fptr) {
    cgCallHelper(m_as, (TCA)fptr, dst, kNoSyncPoint,
                 ArgGroup(m_regs).ssa(tl).ssa(tr));
  } else {
    if (lType.subtypeOf(Type::Obj) || lType.needsReg() ||
        rType.subtypeOf(Type::Obj) || rType.needsReg()) {
      CG_PUNT(cgConcat);
    }
    cgCallHelper(m_as, (TCA)concat_value, dst, kNoSyncPoint,
                 ArgGroup(m_regs).typedValue(tl).typedValue(tr));
  }
}

void CodeGenerator::cgInterpOne(IRInstruction* inst) {
  SSATmp* fp = inst->src(0);
  SSATmp* sp = inst->src(1);
  SSATmp* pcOffTmp  = inst->src(2);
  SSATmp* spAdjustmentTmp = inst->src(3);
  int64_t pcOff = pcOffTmp->getValInt();

  auto opc = *(curFunc()->unit()->at(pcOff));
  void* interpOneHelper = interpOneEntryPoints[opc];

  auto dstReg = InvalidReg;
  cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
               ArgGroup(m_regs).ssa(fp).ssa(sp).imm(pcOff));

  auto newSpReg = m_regs[inst->dst()].reg();
  assert(newSpReg == m_regs[sp].reg());

  int64_t spAdjustBytes = cellsToBytes(spAdjustmentTmp->getValInt());
  if (spAdjustBytes != 0) {
    m_as.addq(spAdjustBytes, newSpReg);
  }
}

void CodeGenerator::cgInterpOneCF(IRInstruction* inst) {
  SSATmp* fp = inst->src(0);
  SSATmp* sp = inst->src(1);
  int64_t pcOff = inst->src(2)->getValInt();

  auto opc = *(curFunc()->unit()->at(pcOff));
  void* interpOneHelper = interpOneEntryPoints[opc];

  auto dstReg = InvalidReg;
  cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
               ArgGroup(m_regs).ssa(fp).ssa(sp).imm(pcOff));

  // The interpOne method returns a pointer to the current ExecutionContext
  // in rax.  Use it read the 'm_fp' and 'm_stack.m_top' fields into the
  // rVmFp and rVmSp registers.
  m_as.loadq(rax[offsetof(VMExecutionContext, m_fp)], rVmFp);
  m_as.loadq(rax[offsetof(VMExecutionContext, m_stack) +
                 Stack::topOfStackOffset()], rVmSp);

  m_tx64->emitServiceReq(SRFlags::EmitInA, REQ_RESUME, 0ull);
}

void CodeGenerator::cgContEnter(IRInstruction* inst) {
  auto contAR = inst->src(0);
  auto addr = inst->src(1);
  auto returnOff = inst->src(2);
  auto curFp = m_regs[inst->src(3)].reg();
  auto contARReg = m_regs[contAR].reg();

  m_as.  storel (returnOff->getValInt(), contARReg[AROFF(m_soff)]);
  m_as.  storeq (curFp, contARReg[AROFF(m_savedRbp)]);
  m_as.  movq   (contARReg, rStashedAR);

  m_as.  call   (m_regs[addr].reg());
}

void CodeGenerator::emitContVarEnvHelperCall(SSATmp* fp, TCA helper) {
  auto scratch = m_rScratch;

  m_as.  loadq (m_regs[fp].reg()[AROFF(m_varEnv)], scratch);
  m_as.  testq (scratch, scratch);
  unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
    cgCallHelper(a, helper, InvalidReg, kNoSyncPoint,
                 ArgGroup(m_regs).ssa(fp));
  });
}

void CodeGenerator::cgUnlinkContVarEnv(IRInstruction* inst) {
  emitContVarEnvHelperCall(
    inst->src(0),
    (TCA)VMExecutionContext::packContVarEnvLinkage);
}

void CodeGenerator::cgLinkContVarEnv(IRInstruction* inst) {
  emitContVarEnvHelperCall(
    inst->src(0),
    (TCA)VMExecutionContext::unpackContVarEnvLinkage);
}

void CodeGenerator::cgContPreNext(IRInstruction* inst) {
  auto contReg = m_regs[inst->src(0)].reg();

  const Offset doneOffset = c_Continuation::doneOffset();
  static_assert((doneOffset + 1) == c_Continuation::runningOffset(),
                "done should immediately precede running");
  // Check done and running at the same time
  m_as.test_imm16_disp_reg16(0x0101, doneOffset, contReg);
  emitFwdJcc(CC_NZ, inst->taken());

  // ++m_index
  m_as.add_imm64_disp_reg64(0x1, CONTOFF(m_index), contReg);
  // running = true
  m_as.store_imm8_disp_reg(0x1, c_Continuation::runningOffset(), contReg);
}

void CodeGenerator::cgContStartedCheck(IRInstruction* inst) {
  m_as.cmp_imm64_disp_reg64(0, CONTOFF(m_index),
                            m_regs[inst->src(0)].reg());
  emitFwdJcc(CC_L, inst->taken());
}

void CodeGenerator::cgIterInit(IRInstruction* inst) {
  cgIterInitCommon(inst);
}

void CodeGenerator::cgIterInitK(IRInstruction* inst) {
  cgIterInitCommon(inst);
}

void CodeGenerator::cgWIterInit(IRInstruction* inst) {
  cgIterInitCommon(inst);
}

void CodeGenerator::cgWIterInitK(IRInstruction* inst) {
  cgIterInitCommon(inst);
}

void CodeGenerator::cgIterInitCommon(IRInstruction* inst) {
  bool isInitK = inst->op() == IterInitK || inst->op() == WIterInitK;
  bool isWInit = inst->op() == WIterInit || inst->op() == WIterInitK;

  PhysReg        fpReg = m_regs[inst->src(1)].reg();
  int64_t     iterOffset = this->iterOffset(inst->src(2));
  int64_t valLocalOffset = localOffset(inst->src(3));
  SSATmp*          src = inst->src(0);
  ArgGroup args(m_regs);
  args.addr(fpReg, iterOffset).ssa(src);
  if (src->isArray()) {
    args.addr(fpReg, valLocalOffset);
    if (isInitK) {
      args.addr(fpReg, localOffset(inst->src(4)));
    } else if (isWInit) {
      args.imm(0);
    }
    TCA helperAddr = isWInit ? (TCA)new_iter_array_key<true> :
      isInitK ? (TCA)new_iter_array_key<false> : (TCA)new_iter_array;
    cgCallHelper(m_as, helperAddr, inst->dst(), kSyncPoint, args);
  } else {
    assert(src->type() == Type::Obj);
    args.imm(uintptr_t(curClass())).addr(fpReg, valLocalOffset);
    if (isInitK) {
      args.addr(fpReg, localOffset(inst->src(4)));
    } else {
      args.imm(0);
    }
    // new_iter_object decrefs its src object if it propagates an
    // exception out, so we use kSyncPointAdjustOne, which adjusts the
    // stack pointer by 1 stack element on an unwind, skipping over
    // the src object.
    cgCallHelper(m_as, (TCA)new_iter_object, inst->dst(),
                 kSyncPointAdjustOne, args);
  }
}

void CodeGenerator::cgIterNext(IRInstruction* inst) {
  cgIterNextCommon(inst);
}

void CodeGenerator::cgIterNextK(IRInstruction* inst) {
  cgIterNextCommon(inst);
}

void CodeGenerator::cgWIterNext(IRInstruction* inst) {
  cgIterNextCommon(inst);
}

void CodeGenerator::cgWIterNextK(IRInstruction* inst) {
  cgIterNextCommon(inst);
}

void CodeGenerator::cgIterNextCommon(IRInstruction* inst) {
  bool isNextK = inst->op() == IterNextK || inst->op() == WIterNextK;
  bool isWNext = inst->op() == WIterNext || inst->op() == WIterNextK;
  PhysReg fpReg = m_regs[inst->src(0)].reg();
  ArgGroup args(m_regs);
  args.addr(fpReg, iterOffset(inst->src(1)))
      .addr(fpReg, localOffset(inst->src(2)));
  if (isNextK) {
    args.addr(fpReg, localOffset(inst->src(3)));
  } else if (isWNext) {
    args.imm(0);
  }
  TCA helperAddr = isWNext ? (TCA)iter_next_key<true> :
    isNextK ? (TCA)iter_next_key<false> : (TCA)iter_next;
  cgCallHelper(m_as, helperAddr, inst->dst(), kSyncPoint, args);
}

void iterFreeHelper(Iter* iter) {
  iter->free();
}

void citerFreeHelper(Iter* iter) {
  iter->cfree();
}

void CodeGenerator::cgIterFree(IRInstruction* inst) {
  PhysReg fpReg = m_regs[inst->src(0)].reg();
  int64_t offset = iterOffset(inst->extra<IterFree>()->iterId);
  cgCallHelper(m_as, (TCA)iterFreeHelper, InvalidReg, kSyncPoint,
               ArgGroup(m_regs).addr(fpReg, offset));
}

void CodeGenerator::cgDecodeCufIter(IRInstruction* inst) {
  PhysReg fpReg = m_regs[inst->src(1)].reg();
  int64_t offset = iterOffset(inst->extra<DecodeCufIter>()->iterId);
  cgCallHelper(m_as, (TCA)decodeCufIterHelper, inst->dst(), kSyncPoint,
               ArgGroup(m_regs).addr(fpReg, offset).typedValue(inst->src(0)));
}

void CodeGenerator::cgCIterFree(IRInstruction* inst) {
  PhysReg fpReg = m_regs[inst->src(0)].reg();
  int64_t  offset = iterOffset(inst->extra<CIterFree>()->iterId);
  cgCallHelper(m_as, (TCA)citerFreeHelper, InvalidReg, kSyncPoint,
               ArgGroup(m_regs).addr(fpReg, offset));
}

void CodeGenerator::cgIncStat(IRInstruction *inst) {
  Stats::emitInc(m_as,
                 Stats::StatCounter(inst->src(0)->getValInt()),
                 inst->src(1)->getValInt(),
                 Transl::CC_None,
                 inst->src(2)->getValBool());
}

void CodeGenerator::cgIncTransCounter(IRInstruction* inst) {
  m_tx64->emitTransCounterInc(m_as);
}

void CodeGenerator::cgDbgAssertRefCount(IRInstruction* inst) {
  emitAssertRefCount(m_as, m_regs[inst->src(0)].reg());
}

void traceCallback(ActRec* fp, Cell* sp, int64_t pcOff, void* rip) {
  if (HPHP::Trace::moduleEnabled(HPHP::Trace::hhirTracelets)) {
    FTRACE(0, "{} {} {}\n", fp->m_func->fullName()->data(), pcOff, rip);
  }
  checkFrame(fp, sp, /*checkLocals*/true);
}

void CodeGenerator::cgDbgAssertType(IRInstruction* inst) {
  emitTypeTest(inst->typeParam(),
               m_regs[inst->src(0)].reg(1),
               m_regs[inst->src(0)].reg(0),
               [&](ConditionCode cc) {
                 ifThen(m_as, ccNegate(cc), [&] { m_as.ud2(); });
               });
}

void CodeGenerator::cgVerifyParamCls(IRInstruction* inst) {
  SSATmp* objClass = inst->src(0);
  assert(!objClass->isConst());
  auto objClassReg = m_regs[objClass].reg();
  SSATmp* constraint = inst->src(1);

  if (constraint->isConst()) {
    m_as.  cmpq(constraint->getValClass(), objClassReg);
  } else {
    m_as.  cmpq(m_regs[constraint].reg(), objClassReg);
  }

  // The native call for this instruction is the slow path that does
  // proper subtype checking. The comparison above is just to
  // short-circuit the overhead when the Classes are an exact match.
  ifThen(m_as, CC_NE, [&]{ cgCallNative(inst); });
}

static void emitTraceCall(CodeGenerator::Asm& as,
                          int64_t pcOff,
                          Transl::TranslatorX64* tx64) {
  // call to a trace function
  as.mov_imm64_reg((int64_t)as.code.frontier, reg::rcx);
  as.mov_reg64_reg64(rVmFp, reg::rdi);
  as.mov_reg64_reg64(rVmSp, reg::rsi);
  as.mov_imm64_reg(pcOff, reg::rdx);
  // do the call; may use a trampoline
  tx64->emitCall(as, (TCA)traceCallback);
}

void CodeGenerator::print() const {
  JIT::print(std::cout, m_curTrace, &m_state.regs, m_state.lifetime,
             m_state.asmInfo);
}

static void patchJumps(Asm& as, CodegenState& state, Block* block) {
  void* list = state.patches[block];
  Address labelAddr = as.code.frontier;
  while (list) {
    int32_t* toPatch = (int32_t*)list;
    int32_t diffToNext = *toPatch;
    ssize_t diff = labelAddr - ((Address)list + sizeof(int32_t));
    *toPatch = safe_cast<int32_t>(diff); // patch the jump address
    if (diffToNext == 0) break;
    void* next = (TCA)list - diffToNext;
    list = next;
  }
}

void CodeGenerator::cgBlock(Block* block, vector<TransBCMapping>* bcMap) {
  FTRACE(6, "cgBlock: {}\n", block->id());

  for (IRInstruction& instr : *block) {
    IRInstruction* inst = &instr;
    if (inst->op() == Marker) {
      m_state.lastMarker = inst->extra<Marker>();
      FTRACE(7, "lastMarker is now {}\n", inst->extra<Marker>()->show());
      if (m_tx64 && m_tx64->isTransDBEnabled() && bcMap) {
        bcMap->push_back((TransBCMapping){Offset(m_state.lastMarker->bcOff),
              m_as.code.frontier,
              m_astubs.code.frontier});
      }
    }
    m_curInst = inst;
    auto nuller = folly::makeGuard([&]{ m_curInst = nullptr; });
    auto* addr = cgInst(inst);
    if (m_state.asmInfo && addr) {
      m_state.asmInfo->instRanges[inst] = TcaRange(addr, m_as.code.frontier);
      m_state.asmInfo->asmRanges[block] =
        TcaRange(m_state.asmInfo->asmRanges[block].start(), m_as.code.frontier);
    }
  }
}

/*
 * Compute and save registers that are live *across* each inst, not including
 * registers whose lifetimes end at inst, nor registers defined by inst.
 */
LiveRegs computeLiveRegs(const IRFactory* factory, const RegAllocInfo& regs,
                         Block* start_block) {
  StateVector<Block, RegSet> liveMap(factory, RegSet());
  LiveRegs live_regs(factory, RegSet());
  postorderWalk(
    [&](Block* block) {
      RegSet& live = liveMap[block];
      if (Block* taken = block->taken()) live = liveMap[taken];
      if (Block* next = block->next()) live |= liveMap[next];
      for (auto it = block->end(); it != block->begin(); ) {
        IRInstruction& inst = *--it;
        for (const SSATmp& dst : inst.dsts()) {
          live -= regs[dst].regs();
        }
        live_regs[inst] = live;
        for (SSATmp* src : inst.srcs()) {
          live |= regs[src].regs();
        }
      }
    },
    factory->numBlocks(),
    start_block
  );
  return live_regs;
}

void genCodeForTrace(IRTrace* trace,
                     CodeGenerator::Asm& as,
                     CodeGenerator::Asm& astubs,
                     IRFactory* irFactory,
                     vector<TransBCMapping>* bcMap,
                     Transl::TranslatorX64* tx64,
                     const RegAllocInfo& regs,
                     const LifetimeInfo* lifetime,
                     AsmInfo* asmInfo) {
  assert(trace->isMain());
  LiveRegs live_regs = computeLiveRegs(irFactory, regs, trace->front());
  CodegenState state(irFactory, regs, live_regs, lifetime, asmInfo);

  // Returns: whether a block has already been emitted.
  DEBUG_ONLY auto isEmitted = [&](Block* block) {
    return state.addresses[block];
  };

  /*
   * Emit the given block on the supplied assembler.  The `nextBlock'
   * is the nextBlock that will be emitted on this assembler.  If is
   * not the fallthrough block, emit a patchable jump to the
   * fallthrough block.
   */
  auto emitBlock = [&](Asm& a, Block* block, Block* nextBlock) {
    assert(!isEmitted(block));

    FTRACE(6, "cgBlock {} on {}\n", block->id(),
           &a == &astubs ? "astubs" : "a");

    auto const aStart      = a.code.frontier;
    auto const astubsStart = astubs.code.frontier;
    patchJumps(a, state, block);
    state.addresses[block] = aStart;

    // If the block ends with a Jmp_ and the next block is going to be
    // its target, we don't need to actually emit it.
    IRInstruction* last = block->back();
    state.noTerminalJmp_ = last->op() == Jmp_ && nextBlock == last->taken();

    CodeGenerator cg(trace, a, astubs, tx64, state);
    if (state.asmInfo) {
      state.asmInfo->asmRanges[block] = TcaRange(aStart, a.code.frontier);
    }

    cg.cgBlock(block, bcMap);
    state.lastMarker = nullptr;
    if (auto next = block->next()) {
      if (next != nextBlock) {
        // If there's a fallthrough block and it's not the next thing
        // going into this assembler, then emit a jump to it.
        emitFwdJmp(a, next, state);
      }
    }

    if (state.asmInfo) {
      state.asmInfo->asmRanges[block] = TcaRange(aStart, a.code.frontier);
      if (&a != &astubs) {
        state.asmInfo->astubRanges[block] = TcaRange(astubsStart,
                                                     astubs.code.frontier);
      }
    }
  };

  if (RuntimeOption::EvalHHIRGenerateAsserts && trace->isMain()) {
    emitTraceCall(as, trace->bcOff(), tx64);
  }

  auto const linfo = layoutBlocks(trace, *irFactory);

  for (auto it = linfo.blocks.begin(); it != linfo.astubsIt; ++it) {
    Block* nextBlock = boost::next(it) != linfo.astubsIt
      ? *boost::next(it) : nullptr;
    emitBlock(as, *it, nextBlock);
  }
  for (auto it = linfo.astubsIt; it != linfo.blocks.end(); ++it) {
    Block* nextBlock = boost::next(it) != linfo.blocks.end()
      ? *boost::next(it) : nullptr;
    emitBlock(astubs, *it, nextBlock);
  }

  if (debug) {
    for (Block* UNUSED block : linfo.blocks) {
      assert(isEmitted(block));
    }
  }
}

ALWAYS_INLINE
TypedValue& getDefaultIfNullCell(TypedValue* tv, TypedValue& def) {
  if (UNLIKELY(nullptr == tv)) {
    // refcount is already correct since def was never decrefed
    return def;
  }
  tvRefcountedDecRef(&def);
  TypedValue* ret = tvToCell(tv);
  tvRefcountedIncRef(ret);
  return *ret;
}

HOT_FUNC_VM
TypedValue arrayIdxS(ArrayData* a, StringData* key, TypedValue def) {
  return getDefaultIfNullCell(a->nvGet(key), def);
}

HOT_FUNC_VM
TypedValue arrayIdxSi(ArrayData* a, StringData* key, TypedValue def) {
  int64_t i;
  return UNLIKELY(key->isStrictlyInteger(i)) ?
         getDefaultIfNullCell(a->nvGet(i), def) :
         getDefaultIfNullCell(a->nvGet(key), def);
}

HOT_FUNC_VM
TypedValue arrayIdxI(ArrayData* a, int64_t key, TypedValue def) {
  return getDefaultIfNullCell(a->nvGet(key), def);
}

}}