From 2d3b8e6056936d2bd5d84fc7bf39ddc7f6160a35 Mon Sep 17 00:00:00 2001 From: Edwin Smith Date: Mon, 17 Mar 2014 10:46:30 -0700 Subject: [PATCH] Dont use rAsm implicitly in x64Assembler This removes the implicit uses of rAsm from X64Assembler, and moves the definition to abi-x64.h. Some mentions of rAsm were hoisted, but most involved doing stores, so those were converted to store two 32-bit immediates (without using a scratch register) when possible. The Immed wrapper class now only allows 32-bit immediates, and a new Immed64 wrapper is available for immediates whose compile-time-type (vm-compile time) is 64 bits. Reviewed By: @ottoni Differential Revision: D1224706 --- hphp/runtime/base/datatype.h | 4 +- hphp/runtime/base/ref-data.h | 2 +- hphp/runtime/vm/jit/abi-x64.h | 14 +- hphp/runtime/vm/jit/arg-group.cpp | 10 +- hphp/runtime/vm/jit/arg-group.h | 33 ++-- hphp/runtime/vm/jit/code-gen-arm.cpp | 8 +- hphp/runtime/vm/jit/code-gen-arm.h | 4 +- hphp/runtime/vm/jit/code-gen-helpers-x64.cpp | 10 +- hphp/runtime/vm/jit/code-gen-helpers-x64.h | 12 ++ hphp/runtime/vm/jit/code-gen-x64.cpp | 221 ++++++++++++++++----------- hphp/runtime/vm/jit/code-gen-x64.h | 4 +- hphp/runtime/vm/jit/debug-guards.cpp | 9 +- hphp/runtime/vm/jit/extra-data.h | 2 +- hphp/runtime/vm/jit/func-prologues-x64.cpp | 16 +- hphp/runtime/vm/jit/ir-translator.cpp | 3 - hphp/runtime/vm/jit/phys-reg.cpp | 2 +- hphp/runtime/vm/jit/phys-reg.h | 4 +- hphp/runtime/vm/jit/reg-alloc.cpp | 17 ++- hphp/runtime/vm/jit/service-requests-x64.cpp | 10 +- hphp/runtime/vm/jit/translator-inline.h | 9 +- hphp/runtime/vm/jit/type.h | 4 +- hphp/runtime/vm/jit/unique-stubs-x64.cpp | 7 +- hphp/runtime/vm/native.h | 2 +- hphp/util/asm-x64.h | 71 +++------ hphp/util/immed.h | 53 +++++-- hphp/util/test/asm.cpp | 3 +- 26 files changed, 298 insertions(+), 236 deletions(-) diff --git a/hphp/runtime/base/datatype.h b/hphp/runtime/base/datatype.h index b0483f2a17a..1f0a1dbb33b 100644 --- a/hphp/runtime/base/datatype.h +++ b/hphp/runtime/base/datatype.h @@ -235,8 +235,8 @@ inline DataType getDataTypeValue(unsigned index) { } // These are used in type_variant.cpp and mc-generator.cpp -const unsigned int kShiftDataTypeToDestrIndex = 4; -const unsigned int kDestrTableSize = 6; +const int kShiftDataTypeToDestrIndex = 4; +const int kDestrTableSize = 6; #define TYPE_TO_DESTR_IDX(t) ((t) >> kShiftDataTypeToDestrIndex) diff --git a/hphp/runtime/base/ref-data.h b/hphp/runtime/base/ref-data.h index 53741403b3b..a2b14970445 100644 --- a/hphp/runtime/base/ref-data.h +++ b/hphp/runtime/base/ref-data.h @@ -127,7 +127,7 @@ struct RefData { not_reached(); #endif } - static constexpr ptrdiff_t tvOffset() { return offsetof(RefData, m_tv); } + static constexpr int tvOffset() { return offsetof(RefData, m_tv); } void assertValid() const { assert(m_magic == Magic::kMagic); diff --git a/hphp/runtime/vm/jit/abi-x64.h b/hphp/runtime/vm/jit/abi-x64.h index bd6a619ac12..976fbeaa09a 100644 --- a/hphp/runtime/vm/jit/abi-x64.h +++ b/hphp/runtime/vm/jit/abi-x64.h @@ -58,6 +58,11 @@ constexpr PhysReg rVmSp = reg::rbx; */ constexpr PhysReg rVmTl = reg::r12; +/* + * scratch register + */ +constexpr Reg64 rAsm = reg::r10; + ////////////////////////////////////////////////////////////////////// /* * Registers used during a tracelet for program locations. @@ -78,8 +83,7 @@ const RegSet kCallerSaved = RegSet() | RegSet(reg::rdi) | RegSet(reg::r8) | RegSet(reg::r9) - // r10 is reserved for the assembler (rAsm), and for - // various extremely-specific scratch uses + // r10 is for extremely-specific scratch uses (rAsm) // r11 is reserved for CodeGenerator (rCgGP) // // ------------- @@ -202,9 +206,9 @@ const RegSet kAllX64Regs = RegSet(kAllRegs).add(reg::r10) * Some data structures are accessed often enough from translated code * that we have shortcuts for getting offsets into them. */ -#define TVOFF(nm) offsetof(TypedValue, nm) -#define AROFF(nm) offsetof(ActRec, nm) -#define CONTOFF(nm) offsetof(c_Continuation, nm) +#define TVOFF(nm) int(offsetof(TypedValue, nm)) +#define AROFF(nm) int(offsetof(ActRec, nm)) +#define CONTOFF(nm) int(offsetof(c_Continuation, nm)) ////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/vm/jit/arg-group.cpp b/hphp/runtime/vm/jit/arg-group.cpp index 1c0a96d8c21..2bd2c306656 100644 --- a/hphp/runtime/vm/jit/arg-group.cpp +++ b/hphp/runtime/vm/jit/arg-group.cpp @@ -22,14 +22,14 @@ namespace HPHP { namespace JIT { TRACE_SET_MOD(hhir); ArgDesc::ArgDesc(SSATmp* tmp, const PhysLoc& loc, bool val) - : m_imm(-1), m_zeroExtend(false), m_done(false) { + : m_zeroExtend(false), m_done(false) { if (tmp->isConst()) { // tmp is a constant m_srcReg = InvalidReg; if (val) { - m_imm = tmp->type() <= Type::Null ? 0 : tmp->rawVal(); + m_imm64 = tmp->type() <= Type::Null ? 0 : tmp->rawVal(); } else { - m_imm = toDataTypeForCall(tmp->type()); + m_imm64 = toDataTypeForCall(tmp->type()); } m_kind = Kind::Imm; return; @@ -37,7 +37,6 @@ ArgDesc::ArgDesc(SSATmp* tmp, const PhysLoc& loc, bool val) if (val) { assert(loc.reg(0) != InvalidReg); m_srcReg = loc.reg(0); - m_imm = 0; m_kind = Kind::Reg; // zero extend any boolean value that we pass to the helper in case // the helper expects it (e.g., as TypedValue) @@ -47,7 +46,6 @@ ArgDesc::ArgDesc(SSATmp* tmp, const PhysLoc& loc, bool val) if (tmp->numWords() > 1) { assert(loc.reg(1) != InvalidReg); m_srcReg = loc.reg(1); - m_imm = 0; // Since val is false then we're passing tmp's type. TypeReg lets // CodeGenerator know that the value might require some massaging // to be in the right format for the call. @@ -56,7 +54,7 @@ ArgDesc::ArgDesc(SSATmp* tmp, const PhysLoc& loc, bool val) } // arg is the (constant) type of a known-typed value. m_srcReg = InvalidReg; - m_imm = toDataTypeForCall(tmp->type()); + m_imm64 = toDataTypeForCall(tmp->type()); m_kind = Kind::Imm; } diff --git a/hphp/runtime/vm/jit/arg-group.h b/hphp/runtime/vm/jit/arg-group.h index b2b25912fd5..22235f83c09 100644 --- a/hphp/runtime/vm/jit/arg-group.h +++ b/hphp/runtime/vm/jit/arg-group.h @@ -59,8 +59,8 @@ public: Reg, // Normal register TypeReg, // TypedValue's m_type field. Might need arch-specific // mangling before call depending on TypedValue's layout. - Imm, // Immediate - Addr, // Address + Imm, // 64-bit Immediate + Addr, // Address (register plus 32-bit displacement) None, // Nothing: register will contain garbage }; @@ -68,19 +68,29 @@ public: PhysReg srcReg() const { return m_srcReg; } Kind kind() const { return m_kind; } void setDstReg(PhysReg reg) { m_dstReg = reg; } - Immed imm() const { return m_imm; } - bool isZeroExtend() const {return m_zeroExtend;} + Immed64 imm() const { assert(m_kind == Kind::Imm); return m_imm64; } + Immed disp() const { assert(m_kind == Kind::Addr); return m_disp32; } + bool isZeroExtend() const { return m_zeroExtend; } bool done() const { return m_done; } void markDone() { m_done = true; } private: // These should be created using ArgGroup. friend struct ArgGroup; - explicit ArgDesc(Kind kind, PhysReg srcReg, Immed immVal) + explicit ArgDesc(Kind kind, Immed64 imm) + : m_kind(kind) + , m_srcReg(InvalidReg) + , m_dstReg(reg::noreg) + , m_imm64(imm) + , m_zeroExtend(false) + , m_done(false) + {} + + explicit ArgDesc(Kind kind, PhysReg srcReg, Immed disp) : m_kind(kind) , m_srcReg(srcReg) , m_dstReg(reg::noreg) - , m_imm(immVal) + , m_disp32(disp) , m_zeroExtend(false) , m_done(false) {} @@ -91,7 +101,10 @@ private: Kind m_kind; PhysReg m_srcReg; PhysReg m_dstReg; - Immed m_imm; + union { + Immed64 m_imm64; // 64-bit plain immediate + Immed m_disp32; // 32-bit displacement + }; bool m_zeroExtend; bool m_done; }; @@ -132,8 +145,8 @@ struct ArgGroup { return m_stkArgs[i]; } - ArgGroup& imm(uintptr_t imm) { - push_arg(ArgDesc(ArgDesc::Kind::Imm, InvalidReg, imm)); + ArgGroup& imm(Immed64 imm) { + push_arg(ArgDesc(ArgDesc::Kind::Imm, imm)); return *this; } @@ -148,7 +161,7 @@ struct ArgGroup { return *this; } - ArgGroup& addr(PhysReg base, intptr_t off) { + ArgGroup& addr(PhysReg base, Immed off) { push_arg(ArgDesc(ArgDesc::Kind::Addr, base, off)); return *this; } diff --git a/hphp/runtime/vm/jit/code-gen-arm.cpp b/hphp/runtime/vm/jit/code-gen-arm.cpp index a3d21ae1eb4..bd7e7e154c3 100644 --- a/hphp/runtime/vm/jit/code-gen-arm.cpp +++ b/hphp/runtime/vm/jit/code-gen-arm.cpp @@ -673,7 +673,7 @@ void CodeGenerator::emitDecRefStaticType(Type type, } void CodeGenerator::emitDecRefDynamicType(vixl::Register baseReg, - ptrdiff_t offset) { + int offset) { // Make sure both temp registers are still available assert(!baseReg.Is(rAsm)); assert(!baseReg.Is(rAsm2)); @@ -712,7 +712,7 @@ void CodeGenerator::emitDecRefDynamicType(vixl::Register baseReg, void CodeGenerator::emitDecRefMem(Type type, vixl::Register baseReg, - ptrdiff_t offset) { + int offset) { if (type.needsReg()) { emitDecRefDynamicType(baseReg, offset); } else if (type.maybeCounted()) { @@ -974,7 +974,7 @@ static void shuffleArgs(vixl::MacroAssembler& a, if (argDesc) { auto kind = argDesc->kind(); if (kind == ArgDesc::Kind::Addr) { - emitRegGetsRegPlusImm(a, dstReg, srcReg, argDesc->imm().q()); + emitRegGetsRegPlusImm(a, dstReg, srcReg, argDesc->disp().l()); } else { if (argDesc->isZeroExtend()) { // "Unsigned eXTend Byte". The dest reg is a 32-bit reg but this @@ -1538,7 +1538,7 @@ void CodeGenerator::cgCallBuiltin(IRInstruction* inst) { auto numArgs = args.size(); DataType funcReturnType = func->returnType(); - ptrdiff_t returnOffset = MISOFF(tvBuiltinReturn); + int returnOffset = MISOFF(tvBuiltinReturn); if (FixupMap::eagerRecord(func)) { // Save VM registers diff --git a/hphp/runtime/vm/jit/code-gen-arm.h b/hphp/runtime/vm/jit/code-gen-arm.h index af418eb0266..add5279fe77 100644 --- a/hphp/runtime/vm/jit/code-gen-arm.h +++ b/hphp/runtime/vm/jit/code-gen-arm.h @@ -93,9 +93,9 @@ struct CodeGenerator { SyncOptions sync, ArgGroup& args); - void emitDecRefDynamicType(vixl::Register baseReg, ptrdiff_t offset); + void emitDecRefDynamicType(vixl::Register baseReg, int offset); void emitDecRefStaticType(Type type, vixl::Register reg); - void emitDecRefMem(Type type, vixl::Register baseReg, ptrdiff_t offset); + void emitDecRefMem(Type type, vixl::Register baseReg, int offset); template void emitTypeTest(Type type, vixl::Register typeReg, Loc dataSrc, diff --git a/hphp/runtime/vm/jit/code-gen-helpers-x64.cpp b/hphp/runtime/vm/jit/code-gen-helpers-x64.cpp index f7787edf95c..a2e486cb96a 100644 --- a/hphp/runtime/vm/jit/code-gen-helpers-x64.cpp +++ b/hphp/runtime/vm/jit/code-gen-helpers-x64.cpp @@ -79,16 +79,12 @@ void emitEagerSyncPoint(Asm& as, const Op* pc) { static COff fpOff = offsetof(ExecutionContext, m_fp); static COff pcOff = offsetof(ExecutionContext, m_pc); - /* we can't use rAsm because the pc store uses it as a - temporary */ - Reg64 rEC = reg::rdi; - - as. push(rEC); + // we can use rAsm because we don't clobber it in X64Assembler + Reg64 rEC = rAsm; emitGetGContext(as, rEC); as. storeq(rVmFp, rEC[fpOff]); as. storeq(rVmSp, rEC[spOff]); - as. storeq(pc, rEC[pcOff]); - as. pop(rEC); + emitImmStoreq(as, intptr_t(pc), rEC[pcOff]); } // emitEagerVMRegSave -- diff --git a/hphp/runtime/vm/jit/code-gen-helpers-x64.h b/hphp/runtime/vm/jit/code-gen-helpers-x64.h index 4b487328c66..db81edaab19 100644 --- a/hphp/runtime/vm/jit/code-gen-helpers-x64.h +++ b/hphp/runtime/vm/jit/code-gen-helpers-x64.h @@ -65,6 +65,18 @@ void emitLdClsCctx(Asm& as, PhysReg srcReg, PhysReg dstReg); void emitCall(Asm& as, TCA dest); void emitCall(Asm& as, CppCall call); +// store imm to the 8-byte memory location at ref. Warning: don't use this +// if you wanted an atomic store; large imms cause two stores. +template +void emitImmStoreq(Asm& as, Immed64 imm, Ref ref) { + if (imm.fits(sz::dword)) { + as.storeq(imm.l(), ref); // sign-extend to 64-bit then storeq + } else { + as.storel(int32_t(imm.q()), ref); + as.storel(int32_t(imm.q() >> 32), Ref(ref.r + 4)); + } +} + void emitJmpOrJcc(Asm& as, ConditionCode cc, TCA dest); void emitRB(Asm& a, Trace::RingBufferType t, const char* msgm, diff --git a/hphp/runtime/vm/jit/code-gen-x64.cpp b/hphp/runtime/vm/jit/code-gen-x64.cpp index 78ecbc35159..1d35f1a8738 100644 --- a/hphp/runtime/vm/jit/code-gen-x64.cpp +++ b/hphp/runtime/vm/jit/code-gen-x64.cpp @@ -437,11 +437,11 @@ void CodeGenerator::emitCompare(IRInstruction* inst) { auto reg0 = loc0.reg(); auto reg1 = loc1.reg(); - if (src1->isConst()) { + if (reg1 == InvalidReg) { if (type0 <= Type::Bool) { - m_as. cmpb (src1->rawVal(), rbyte(reg0)); + m_as. cmpb (src1->boolVal(), rbyte(reg0)); } else { - m_as. cmpq (src1->rawVal(), reg0); + m_as. cmpq (safe_cast(src1->intVal()), reg0); } } else { // Note the reverse syntax in the assembler. @@ -459,7 +459,7 @@ void CodeGenerator::emitCompareInt(IRInstruction* inst) { auto srcReg0 = srcLoc(0).reg(); auto srcReg1 = srcLoc(1).reg(); if (srcReg1 == InvalidReg) { - m_as. cmpq(inst->src(1)->rawVal(), srcReg0); + m_as. cmpq(safe_cast(inst->src(1)->intVal()), srcReg0); } else { // Note the reverse syntax in the assembler. // This cmp will compute srcReg0 - srcReg1 @@ -750,14 +750,14 @@ static int64_t shuffleArgs(Asm& a, ArgGroup& args, CppCall& call) { break; case ArgDesc::Kind::Addr: - a. lea (arg.srcReg()[arg.imm().l()], rTmp); + a. lea (arg.srcReg()[arg.disp().l()], rTmp); a. push(rTmp); break; case ArgDesc::Kind::None: a. push(rax); if (RuntimeOption::EvalHHIRGenerateAsserts) { - a. storeq(0xbadbadbadbadbad, *rsp); + emitImmStoreq(a, 0xbadbadbadbadbad, *rsp); } break; } @@ -789,7 +789,7 @@ static int64_t shuffleArgs(Asm& a, ArgGroup& args, CppCall& call) { assert(kind == ArgDesc::Kind::Addr); assert(how.m_src.isGP()); assert(how.m_dst.isGP()); - a. lea (how.m_src[argDesc->imm().q()], how.m_dst); + a. lea (how.m_src[argDesc->disp().l()], how.m_dst); } if (kind != ArgDesc::Kind::TypeReg) { argDesc->markDone(); @@ -822,7 +822,7 @@ static int64_t shuffleArgs(Asm& a, ArgGroup& args, CppCall& call) { a. shlq (kTypeShiftBits, dst); } } else if (kind == ArgDesc::Kind::Addr) { - a. addq (args[i].imm(), dst); + a. addq (args[i].disp(), dst); } else if (args[i].isZeroExtend()) { a. movzbl (rbyte(dst), r32(dst)); } else if (RuntimeOption::EvalHHIRGenerateAsserts && @@ -1033,6 +1033,8 @@ void CodeGenerator::cgAbsDbl(IRInstruction* inst) { inline static Reg8 convertToReg8(PhysReg reg) { return rbyte(reg); } inline static Reg64 convertToReg64(PhysReg reg) { return reg; } +typedef void (Asm::*AsmInstrIR)(Immed, Reg64); + template void CodeGenerator::cgBinaryIntOp(IRInstruction* inst, void (Asm::*instrIR)(Immed, RegType), @@ -1083,15 +1085,20 @@ void CodeGenerator::cgBinaryIntOp(IRInstruction* inst, return; } + auto intVal = [] (const SSATmp* s) { + return s->isA(Type::Int) ? safe_cast(s->intVal()) : + int32_t(s->boolVal()); + }; + // One register, and one immediate. if (commutative) { - auto immedSrc = (src2Reg == InvalidReg ? src2 : src1); - auto immed = immedSrc->rawVal(); + assert(instrIR); + auto imm = intVal(src2Reg == InvalidReg ? src2 : src1); auto srcReg = srcLoc(src2Reg == InvalidReg ? 0 : 1).reg(); if (srcReg == dstReg) { - (a.*instrIR) (immed, dstOpReg); + (a.*instrIR) (imm, dstOpReg); } else { - emitLoadImm(a, immed, dstReg); + a. emitImmReg(imm, dstReg); (a.*instrRR) (convertReg(srcReg), dstOpReg); } return; @@ -1100,11 +1107,11 @@ void CodeGenerator::cgBinaryIntOp(IRInstruction* inst, // NonCommutative: if (src1Reg == InvalidReg) { if (dstReg == src2Reg) { - emitLoadImm(a, src1->rawVal(), m_rScratch); + emitLoadImm(a, intVal(src1), m_rScratch); (a.*instrRR) (src2OpReg, rOpScratch); (a.*movInstr)(rOpScratch, dstOpReg); } else { - emitLoadImm(a, src1->rawVal(), dstReg); + emitLoadImm(a, intVal(src1), dstReg); (a.*instrRR) (src2OpReg, dstOpReg); } return; @@ -1112,7 +1119,7 @@ void CodeGenerator::cgBinaryIntOp(IRInstruction* inst, assert(src2Reg == InvalidReg); emitMovRegReg(a, src1Reg, dstReg); - (a.*instrIR) (src2->rawVal(), dstOpReg); + (a.*instrIR) (intVal(src2), dstOpReg); } void CodeGenerator::cgBinaryDblOp(IRInstruction* inst, @@ -1245,7 +1252,7 @@ void CodeGenerator::cgSubInt(IRInstruction* inst) { void CodeGenerator::cgMulInt(IRInstruction* inst) { cgBinaryIntOp( inst, - &Asm::imul, + (AsmInstrIR)nullptr, // there is no imul-immediate instruction &Asm::imul, &Asm::movq, &convertToReg64, @@ -1404,7 +1411,7 @@ void CodeGenerator::cgShiftCommon(IRInstruction* inst, // one immediate (right hand side) if (srcReg1 == InvalidReg) { emitMovRegReg(m_as, srcReg0, dstReg); - (m_as.*instrIR)(inst->src(1)->intVal(), dstReg); + (m_as.*instrIR)(safe_cast(inst->src(1)->intVal()), dstReg); return; } @@ -1793,8 +1800,16 @@ void CodeGenerator::emitSpecializedTypeTest(Type type, DataLoc dataSrc, if (type < Type::Obj) { // emit the specific class test assert(type.getClass()->attrs() & AttrFinal); + auto clsImm = Immed64(type.getClass()); auto reg = getDataPtrEnregistered(m_as, dataSrc, m_rScratch); - m_as.cmpq(type.getClass(), reg[ObjectData::getVMClassOffset()]); + if (clsImm.fits(sz::dword)) { + m_as.cmpq(clsImm.l(), reg[ObjectData::getVMClassOffset()]); + } else { + // use a scratch. We could do this without rAsm using two immediate + // 32-bit compares (and two branches). + m_as.emitImmReg(clsImm, rAsm); + m_as.cmpq(rAsm, reg[ObjectData::getVMClassOffset()]); + } doJcc(CC_E); } else { assert(type < Type::Arr); @@ -2063,7 +2078,7 @@ void CodeGenerator::cgExtendsClass(IRInstruction* inst) { // least as long as the potential base (testClass) it might be a // subclass. asm_label(a, notExact); - a. cmpl (testClass->classVecLen(), + a. cmpl (safe_cast(testClass->classVecLen()), rObjClass[Class::classVecLenOff()]); a. jb8 (falseLabel); @@ -2082,13 +2097,15 @@ void CodeGenerator::cgConvDblToInt(IRInstruction* inst) { auto src = inst->src(0); auto srcReg = prepXMMReg(m_as, src, srcLoc(0), rCgXMM0); auto dstReg = dstLoc(0).reg(); + auto rIndef = rAsm; // not clobbered by emitLoadImm() - constexpr uint64_t indefiniteInteger = 0x8000000000000000LL; constexpr uint64_t maxULongAsDouble = 0x43F0000000000000LL; constexpr uint64_t maxLongAsDouble = 0x43E0000000000000LL; + m_as. emitImmReg (1, rIndef); + m_as. rorq (1, rIndef); // rIndef = 0x8000000000000000 m_as. cvttsd2siq (srcReg, dstReg); - m_as. cmpq (indefiniteInteger, dstReg); + m_as. cmpq (rIndef, dstReg); unlikelyIfBlock(CC_E, [&] (Asm& a) { // result > max signed int or unordered @@ -2122,7 +2139,7 @@ void CodeGenerator::cgConvDblToInt(IRInstruction* inst) { // and flip its sign bit (NB: we don't use orq here because it's // possible that src0 == LONG_MAX in which case cvttsd2siq will yeild // an indefiniteInteger, which we would like to make zero) - a. xorq (indefiniteInteger, dstReg); + a. xorq (rIndef, dstReg); }); asm_label(a, isUnordered); @@ -2184,7 +2201,7 @@ void testimm(Asm& as, uint32_t val, const M& mr) { if (v > 0xff) { as.testl((int32_t)val, mr); } else { - as.testb(v, *(mr.r + off)); + as.testb((int8_t)v, *(mr.r + off)); } } @@ -2450,7 +2467,7 @@ asm_label(a, slow_path); kVoidDest, SyncOptions::kSmashableAndSyncPoint, argGroup() - .addr(rVmTl, handle) + .addr(rVmTl, safe_cast(handle)) .ssa(2/*actRec*/) .ssa(1/*name*/) .ssa(0/*cls*/) @@ -2546,11 +2563,17 @@ void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) { JmpSwitchData* data = inst->extra(); SSATmp* index = inst->src(0); auto indexReg = srcLoc(0).reg(); + auto rTmp = m_rScratch; if (!index->isConst()) { if (data->bounded) { if (data->base) { - m_as. subq(data->base, indexReg); + if (deltaFits(data->base, sz::dword)) { + m_as. subq(safe_cast(data->base), indexReg); + } else { + m_as. emitImmReg(data->base, rTmp); + m_as. subq(rTmp, indexReg); + } } m_as. cmpq(data->cases - 2, indexReg); prepareForSmash(m_mainCode, kJmpccLen); @@ -2665,9 +2688,8 @@ void CodeGenerator::cgDefInlineFP(IRInstruction* inst) { auto const fakeRet = m_mcg->tx().uniqueStubs.retInlHelper; auto const retBCOff = inst->extra()->retBCOff; - m_as. storeq (fakeRet, fp[AROFF(m_savedRip)]); + emitImmStoreq(m_as, intptr_t(fakeRet), fp[AROFF(m_savedRip)]); m_as. storel (retBCOff, fp[AROFF(m_soff)]); - cgMov(inst); } @@ -3280,7 +3302,7 @@ void CodeGenerator::cgCufIterSpillFrame(IRInstruction* inst) { auto const iterId = inst->extra()->iterId; auto const itOff = iterOffset(iterId); - const int64_t spOffset = -kNumActRecCells * sizeof(Cell); + const auto spOffset = -safe_cast(kNumActRecCells * sizeof(Cell)); auto spReg = srcLoc(0).reg(); auto fpReg = srcLoc(1).reg(); @@ -3305,7 +3327,8 @@ void CodeGenerator::cgCufIterSpillFrame(IRInstruction* inst) { }); m_as.storeq (m_rScratch, spReg[spOffset + int(AROFF(m_invName))]); m_as.storeq (fpReg, spReg[spOffset + int(AROFF(m_savedRbp))]); - m_as.storel (nArgs, spReg[spOffset + int(AROFF(m_numArgsAndGenCtorFlags))]); + m_as.storel (safe_cast(nArgs), + spReg[spOffset + int(AROFF(m_numArgsAndGenCtorFlags))]); emitAdjustSp(spReg, dstLoc(0).reg(), spOffset); } @@ -3317,15 +3340,15 @@ void CodeGenerator::cgSpillFrame(IRInstruction* inst) { auto const nArgs = inst->extra()->numArgs; auto& a = m_as; - const int64_t spOffset = -kNumActRecCells * sizeof(Cell); + const auto spOffset = -safe_cast(kNumActRecCells * sizeof(Cell)); auto spReg = srcLoc(0).reg(); // actRec->m_this if (objOrCls->isA(Type::Cls)) { // store class if (objOrCls->isConst()) { - a. storeq (uintptr_t(objOrCls->clsVal()) | 1, - spReg[spOffset + int(AROFF(m_this))]); + emitImmStoreq(a, uintptr_t(objOrCls->clsVal()) | 1, + spReg[spOffset + int(AROFF(m_this))]); } else { Reg64 clsPtrReg = srcLoc(3/*objOrCls*/).reg(); a. movq (clsPtrReg, m_rScratch); @@ -3352,14 +3375,14 @@ void CodeGenerator::cgSpillFrame(IRInstruction* inst) { uintptr_t invName = !magicName ? 0 : reinterpret_cast(magicName) | ActRec::kInvNameBit; - a. storeq (invName, spReg[spOffset + int(AROFF(m_invName))]); + emitImmStoreq(a, invName, spReg[spOffset + int(AROFF(m_invName))]); // actRec->m_func and possibly actRec->m_cls // Note m_cls is unioned with m_this and may overwrite previous value if (func->isA(Type::Null)) { assert(func->isConst()); } else if (func->isConst()) { const Func* f = func->funcVal(); - a. storeq (f, spReg[spOffset + int(AROFF(m_func))]); + emitImmStoreq(a, intptr_t(f), spReg[spOffset + int(AROFF(m_func))]); } else { int offset_m_func = spOffset + int(AROFF(m_func)); auto funcLoc = srcLoc(2); @@ -3370,16 +3393,13 @@ void CodeGenerator::cgSpillFrame(IRInstruction* inst) { a. storeq (fpLoc.reg(), spReg[spOffset + int(AROFF(m_savedRbp))]); a. storel (nArgs, spReg[spOffset + int(AROFF(m_numArgsAndGenCtorFlags))]); - emitAdjustSp(spReg, - dstLoc(0).reg(), - spOffset); + emitAdjustSp(spReg, dstLoc(0).reg(), spOffset); } void CodeGenerator::cgStClosureFunc(IRInstruction* inst) { auto const obj = srcLoc(0).reg(); auto const func = inst->extra()->func; - auto& a = m_as; - a. storeq (func, obj[c_Closure::funcOffset()]); + emitImmStoreq(m_as, intptr_t(func), obj[c_Closure::funcOffset()]); } void CodeGenerator::cgStClosureArg(IRInstruction* inst) { @@ -3414,7 +3434,8 @@ void CodeGenerator::emitInitObjProps(PhysReg dstReg, auto propDataOffset = propOffset + TVOFF(m_data); auto propTypeOffset = propOffset + TVOFF(m_type); if (!IS_NULL_TYPE(cls->declPropInit()[i].m_type)) { - m_as.storeq(cls->declPropInit()[i].m_data.num, dstReg[propDataOffset]); + emitImmStoreq(m_as, cls->declPropInit()[i].m_data.num, + dstReg[propDataOffset]); } m_as.storeb(cls->declPropInit()[i].m_type, dstReg[propTypeOffset]); } @@ -3423,7 +3444,8 @@ void CodeGenerator::emitInitObjProps(PhysReg dstReg, // Use memcpy for large numbers of properties. auto args = argGroup() - .addr(dstReg, sizeof(ObjectData) + cls->builtinODTailSize()) + .addr(dstReg, + safe_cast(sizeof(ObjectData) + cls->builtinODTailSize())) .imm(int64_t(&cls->declPropInit()[0])) .imm(cellsToBytes(nProps)); cgCallHelper(m_as, @@ -3507,7 +3529,8 @@ void CodeGenerator::cgInitObjProps(IRInstruction* inst) { m_as.loadq(rPropData[Class::PropInitVec::dataOff()], rPropData); if (!cls->hasDeepInitProps()) { auto args = argGroup() - .addr(srcReg, sizeof(ObjectData) + cls->builtinODTailSize()) + .addr(srcReg, + safe_cast(sizeof(ObjectData) + cls->builtinODTailSize())) .reg(rPropData) .imm(cellsToBytes(nProps)); cgCallHelper(m_as, @@ -3517,7 +3540,8 @@ void CodeGenerator::cgInitObjProps(IRInstruction* inst) { args); } else { auto args = argGroup() - .addr(srcReg, sizeof(ObjectData) + cls->builtinODTailSize()) + .addr(srcReg, + safe_cast(sizeof(ObjectData) + cls->builtinODTailSize())) .reg(rPropData) .imm(nProps); cgCallHelper(m_as, @@ -3549,15 +3573,15 @@ void CodeGenerator::cgCall(IRInstruction* inst) { SSATmp* returnBcOffset = inst->src(1); SSATmp* func = inst->src(2); SrcRange args = inst->srcs().subpiece(3); - int32_t numArgs = args.size(); + int numArgs = args.size(); // put all outgoing arguments onto the VM stack - int64_t adjustment = (-(int64_t)numArgs) * sizeof(Cell); - for (int32_t i = 0; i < numArgs; i++) { + int adjustment = -numArgs * sizeof(Cell); + for (int i = 0; i < numArgs; i++) { cgStore(spReg[-(i + 1) * sizeof(Cell)], args[i], srcLoc(i+3), Width::Full); } // store the return bytecode offset into the outgoing actrec - uint64_t returnBc = returnBcOffset->intVal(); + auto returnBc = safe_cast(returnBcOffset->intVal()); m_as.storel(returnBc, spReg[AROFF(m_soff)]); if (adjustment != 0) { m_as.addq(adjustment, spReg); @@ -3747,17 +3771,18 @@ void CodeGenerator::cgSpillStack(IRInstruction* inst) { auto const spReg = srcLoc(0).reg(); auto const spillCells = spillValueCells(inst); - int64_t adjustment = (spDeficit - spillCells) * sizeof(Cell); + int adjustment = safe_cast( + (spDeficit - spillCells) * ssize_t(sizeof(Cell)) + ); for (uint32_t i = 0; i < numSpillSrcs; ++i) { - const int64_t offset = i * sizeof(Cell) + adjustment; + int offset = safe_cast(i * ssize_t(sizeof(Cell)) + adjustment); cgStore(spReg[offset], spillVals[i], srcLoc(i + 2), Width::Full); } emitAdjustSp(spReg, dstReg, adjustment); } -void CodeGenerator::emitAdjustSp(PhysReg spReg, - PhysReg dstReg, - int64_t adjustment /* bytes */) { +void CodeGenerator::emitAdjustSp(PhysReg spReg, PhysReg dstReg, + int adjustment /* bytes */) { if (adjustment != 0) { if (dstReg != spReg) { m_as. lea (spReg[adjustment], dstReg); @@ -3872,8 +3897,8 @@ void CodeGenerator::cgStaticLocInitCached(IRInstruction* inst) { a. incl (rdSrc[FAST_REFCOUNT_OFFSET]); if (debug) { static_assert(sizeof(RefData::Magic::kMagic) == sizeof(uint64_t), ""); - a. storeq (static_cast(RefData::Magic::kMagic), - rdSrc[RefData::magicOffset()]); + emitImmStoreq(a, static_cast(RefData::Magic::kMagic), + rdSrc[RefData::magicOffset()]); } } @@ -3908,12 +3933,12 @@ void CodeGenerator::cgStore(BaseRef dst, SSATmp* src, PhysLoc srcLoc, if (src->isA(Type::Null)) return; // no need to store a value for null or // uninit auto memRef = refTVData(dst); - if (src->isConst()) { + auto srcReg = srcLoc.reg(); + if (srcReg == InvalidReg) { always_assert(type <= (Type::Bool | Type::Int | Type::Dbl | Type::Arr | Type::StaticStr | Type::Cls)); - m_as.storeq(src->rawVal(), memRef); + emitImmStoreq(m_as, src->rawVal(), memRef); } else { - auto srcReg = srcLoc.reg(); zeroExtendIfBool(m_as, src, srcReg); emitStoreReg(m_as, srcReg, memRef); } @@ -4084,7 +4109,8 @@ void CodeGenerator::cgStringIsset(IRInstruction* inst) { auto idxReg = srcLoc(1).reg(); auto dstReg = dstLoc(0).reg(); if (idxReg == InvalidReg) { - m_as.cmpl(inst->src(1)->intVal(), strReg[StringData::sizeOff()]); + m_as.cmpl(safe_cast(inst->src(1)->intVal()), + strReg[StringData::sizeOff()]); } else { m_as.cmpl(r32(idxReg), strReg[StringData::sizeOff()]); } @@ -4098,7 +4124,8 @@ void CodeGenerator::cgCheckPackedArrayBounds(IRInstruction* inst) { auto arrReg = srcLoc(0).reg(); auto idxReg = srcLoc(1).reg(); if (idxReg == InvalidReg) { - m_as.cmpl (inst->src(1)->intVal(), arrReg[ArrayData::offsetofSize()]); + m_as.cmpl (safe_cast(inst->src(1)->intVal()), + arrReg[ArrayData::offsetofSize()]); } else { // ArrayData::m_size is a uint32_t but we need to do a 64-bit comparison // since idx is KindOfInt64. @@ -4203,17 +4230,17 @@ void CodeGenerator::cgCheckBounds(IRInstruction* inst) { }; - if (idx->isConst()) { - int64_t idxVal = idx->intVal(); - assert(idxVal >= 0); // we would have punted otherwise - m_as.cmpq(idxVal, srcLoc(1).reg()); + if (idxReg == InvalidReg) { + assert(idx->intVal() >= 0); // we would have punted otherwise + auto idxVal = safe_cast(idx->intVal()); + m_as.cmpq(idxVal, sizeReg); unlikelyIfBlock(CC_LE, throwHelper); return; } - if (size->isConst()) { - int64_t sizeVal = size->intVal(); - assert(sizeVal >= 0); + if (sizeReg == InvalidReg) { + assert(size->intVal() >= 0); + auto sizeVal = safe_cast(size->intVal()); m_as.cmpq(sizeVal, idxReg); } else { m_as.cmpq(sizeReg, idxReg); @@ -4543,7 +4570,13 @@ void CodeGenerator::cgCheckDefinedClsEq(IRInstruction* inst) { auto const clsName = inst->extra()->clsName; auto const cls = inst->extra()->cls; auto const ch = Unit::GetNamedEntity(clsName)->getClassHandle(); - m_as.cmpq(cls, rVmTl[ch]); + auto clsImm = Immed64(cls); + if (clsImm.fits(sz::dword)) { + m_as.cmpq(clsImm.l(), rVmTl[ch]); + } else { + m_as.emitImmReg(cls, m_rScratch); + m_as.cmpq(m_rScratch, rVmTl[ch]); + } emitFwdJcc(CC_NZ, inst->taken()); } @@ -4566,7 +4599,7 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { assert(nParamsReg != InvalidReg || nParamsTmp->isConst()); assert(firstBitNumTmp->isConst(Type::Int)); - uint32_t firstBitNum = (uint32_t)(firstBitNumTmp->intVal()); + auto firstBitNum = safe_cast(firstBitNumTmp->intVal()); uint64_t mask64 = mask64Tmp->intVal(); assert(mask64Reg != InvalidReg || mask64 == uint32_t(mask64)); @@ -4997,7 +5030,7 @@ void CodeGenerator::cgLdClsCached(IRInstruction* inst) { CppCall(func), callDest(inst), SyncOptions::kSyncPoint, - argGroup().addr(rVmTl, intptr_t(ch)) + argGroup().addr(rVmTl, safe_cast(ch)) .ssa(0)); }); } @@ -5072,7 +5105,7 @@ void CodeGenerator::cgLookupClsCns(IRInstruction* inst) { callDestTV(inst), SyncOptions::kSyncPoint, argGroup() - .addr(rVmTl, link.handle()) + .addr(rVmTl, safe_cast(link.handle())) .immPtr(Unit::GetNamedEntity(extra->clsName)) .immPtr(extra->clsName) .immPtr(extra->cnsName) @@ -5096,7 +5129,7 @@ void CodeGenerator::cgLookupCnsCommon(IRInstruction* inst) { auto const ch = makeCnsHandle(cnsName, false); auto args = argGroup(); - args.addr(rVmTl, ch) + args.addr(rVmTl, safe_cast(ch)) .immPtr(cnsName) .imm(inst->op() == LookupCnsE); @@ -5123,7 +5156,7 @@ void CodeGenerator::cgLookupCnsU(IRInstruction* inst) { auto const fallbackCh = makeCnsHandle(fallbackName, false); auto args = argGroup(); - args.addr(rVmTl, fallbackCh) + args.addr(rVmTl, safe_cast(fallbackCh)) .immPtr(cnsName) .immPtr(fallbackName); @@ -5382,7 +5415,7 @@ void CodeGenerator::cgInterpOne(IRInstruction* inst) { auto newSpReg = dstLoc(0).reg(); assert(newSpReg == srcLoc(0).reg()); - int64_t spAdjustBytes = cellsToBytes(extra.cellsPopped - extra.cellsPushed); + auto spAdjustBytes = cellsToBytes(extra.cellsPopped - extra.cellsPushed); if (spAdjustBytes != 0) { m_as.addq(spAdjustBytes, newSpReg); } @@ -5404,7 +5437,7 @@ void CodeGenerator::cgInterpOneCF(IRInstruction* inst) { void CodeGenerator::cgContEnter(IRInstruction* inst) { auto contARReg = srcLoc(0).reg(); auto addrReg = srcLoc(1).reg(); - auto returnOff = inst->src(2)->intVal(); + auto returnOff = safe_cast(inst->src(2)->intVal()); auto curFp = srcLoc(3).reg(); m_as. storel (returnOff, contARReg[AROFF(m_soff)]); @@ -5519,11 +5552,11 @@ void CodeGenerator::emitStRaw(IRInstruction* inst, size_t extraOff) { auto const valueReg = srcLoc(1).reg(); if (valueReg == InvalidReg) { - auto const val = inst->src(0)->rawVal(); + auto val = Immed64(inst->src(0)->rawVal()); switch (size) { - case sz::byte: m_as.storeb(val, dest); break; - case sz::dword: m_as.storel(val, dest); break; - case sz::qword: m_as.storeq(val, dest); break; + case sz::byte: m_as.storeb(val.b(), dest); break; + case sz::dword: m_as.storel(val.l(), dest); break; + case sz::qword: emitImmStoreq(m_as, val.q(), dest); break; default: not_implemented(); } } else { @@ -5632,8 +5665,8 @@ void CodeGenerator::cgIterInitCommon(IRInstruction* inst) { bool isWInit = inst->op() == WIterInit || inst->op() == WIterInitK; PhysReg fpReg = srcLoc(1).reg(); - int64_t iterOffset = this->iterOffset(inst->extra()->iterId); - int64_t valLocalOffset = localOffset(inst->extra()->valId); + int iterOffset = this->iterOffset(inst->extra()->iterId); + int valLocalOffset = localOffset(inst->extra()->valId); SSATmp* src = inst->src(0); auto args = argGroup(); args.addr(fpReg, iterOffset).ssa(0/*src*/); @@ -5675,8 +5708,8 @@ void CodeGenerator::cgMIterInitK(IRInstruction* inst) { void CodeGenerator::cgMIterInitCommon(IRInstruction* inst) { PhysReg fpReg = srcLoc(1).reg(); - int64_t iterOffset = this->iterOffset(inst->extra()->iterId); - int64_t valLocalOffset = localOffset(inst->extra()->valId); + int iterOffset = this->iterOffset(inst->extra()->iterId); + int valLocalOffset = localOffset(inst->extra()->valId); SSATmp* src = inst->src(0); auto args = argGroup(); @@ -5781,7 +5814,7 @@ void CodeGenerator::cgIterCopy(IRInstruction* inst) { void CodeGenerator::cgIterFree(IRInstruction* inst) { PhysReg fpReg = srcLoc(0).reg(); - int64_t offset = iterOffset(inst->extra()->iterId); + int offset = iterOffset(inst->extra()->iterId); cgCallHelper(m_as, CppCall(getMethodPtr(&Iter::free)), kVoidDest, @@ -5791,7 +5824,7 @@ void CodeGenerator::cgIterFree(IRInstruction* inst) { void CodeGenerator::cgMIterFree(IRInstruction* inst) { PhysReg fpReg = srcLoc(0).reg(); - int64_t offset = iterOffset(inst->extra()->iterId); + int offset = iterOffset(inst->extra()->iterId); cgCallHelper(m_as, CppCall(getMethodPtr(&Iter::mfree)), kVoidDest, @@ -5801,7 +5834,7 @@ void CodeGenerator::cgMIterFree(IRInstruction* inst) { void CodeGenerator::cgDecodeCufIter(IRInstruction* inst) { PhysReg fpReg = srcLoc(1).reg(); - int64_t offset = iterOffset(inst->extra()->iterId); + int offset = iterOffset(inst->extra()->iterId); cgCallHelper(m_as, CppCall(decodeCufIterHelper), callDest(inst), SyncOptions::kSyncPoint, argGroup().addr(fpReg, offset) @@ -5810,7 +5843,7 @@ void CodeGenerator::cgDecodeCufIter(IRInstruction* inst) { void CodeGenerator::cgCIterFree(IRInstruction* inst) { PhysReg fpReg = srcLoc(0).reg(); - int64_t offset = iterOffset(inst->extra()->iterId); + int offset = iterOffset(inst->extra()->iterId); cgCallHelper(m_as, CppCall(getMethodPtr(&Iter::cfree)), kVoidDest, @@ -5875,21 +5908,23 @@ void CodeGenerator::cgDbgAssertRetAddr(IRInstruction* inst) { // RetCtrl. always_assert(!inst->is(FreeActRec, RetCtrl)); - m_as.cmpq((uintptr_t)enterTCServiceReq, *rsp); + m_as.cmpq(safe_cast((uintptr_t)enterTCServiceReq), *rsp); ifBlock(CC_NE, [&](Asm& a) { a.ud2(); }); } -void CodeGenerator::cgVerifyClsWork(IRInstruction* inst) { +void CodeGenerator::emitVerifyCls(IRInstruction* inst) { assert(!inst->src(0)->isConst()); auto objClassReg = srcLoc(0).reg(); SSATmp* constraint = inst->src(1); + auto constraintReg = srcLoc(1).reg(); - if (constraint->isConst()) { - m_as. cmpq(constraint->clsVal(), objClassReg); + if (constraintReg == InvalidReg) { + auto imm = Immed64(constraint->clsVal()); + m_as. cmpq(imm.l(), objClassReg); } else { - m_as. cmpq(srcLoc(1).reg(), objClassReg); + m_as. cmpq(constraintReg, objClassReg); } // The native call for this instruction is the slow path that does @@ -5899,11 +5934,11 @@ void CodeGenerator::cgVerifyClsWork(IRInstruction* inst) { } void CodeGenerator::cgVerifyParamCls(IRInstruction* inst) { - cgVerifyClsWork(inst); + emitVerifyCls(inst); } void CodeGenerator::cgVerifyRetCls(IRInstruction* inst) { - cgVerifyClsWork(inst); + emitVerifyCls(inst); } void CodeGenerator::cgRBTrace(IRInstruction* inst) { diff --git a/hphp/runtime/vm/jit/code-gen-x64.h b/hphp/runtime/vm/jit/code-gen-x64.h index f3afa83a766..d40202c13f4 100644 --- a/hphp/runtime/vm/jit/code-gen-x64.h +++ b/hphp/runtime/vm/jit/code-gen-x64.h @@ -145,7 +145,7 @@ private: void (Asm::*instrIR)(Immed, Reg64), void (Asm::*instrR)(Reg64)); - void cgVerifyClsWork(IRInstruction* inst); + void emitVerifyCls(IRInstruction* inst); void emitGetCtxFwdCallWithThis(PhysReg ctxReg, bool staticCallee); @@ -236,7 +236,7 @@ private: int iterOffset(uint32_t id); void emitReqBindAddr(const Func* func, TCA& dest, Offset offset); - void emitAdjustSp(PhysReg spReg, PhysReg dstReg, int64_t adjustment); + void emitAdjustSp(PhysReg spReg, PhysReg dstReg, int adjustment); void emitConvBoolOrIntToDbl(IRInstruction* inst); void cgLdClsMethodCacheCommon(IRInstruction* inst, Offset offset); void emitLdRaw(IRInstruction* inst, size_t extraOff); diff --git a/hphp/runtime/vm/jit/debug-guards.cpp b/hphp/runtime/vm/jit/debug-guards.cpp index 015b3337558..921346c4ed0 100644 --- a/hphp/runtime/vm/jit/debug-guards.cpp +++ b/hphp/runtime/vm/jit/debug-guards.cpp @@ -18,6 +18,7 @@ #include "hphp/util/asm-x64.h" #include "hphp/vixl/a64/macro-assembler-a64.h" #include "hphp/runtime/vm/jit/abi-arm.h" +#include "hphp/runtime/vm/jit/abi-x64.h" #include "hphp/runtime/vm/jit/code-gen-helpers-arm.h" #include "hphp/runtime/vm/jit/code-gen-helpers-x64.h" #include "hphp/runtime/vm/jit/jump-smash.h" @@ -27,7 +28,6 @@ namespace HPHP { namespace JIT { - static constexpr size_t dbgOff = offsetof(ThreadInfo, m_reqInjectionData) + RequestInjectionData::debuggerReadOnlyOffset(); @@ -40,9 +40,10 @@ void addDbgGuardImpl(SrcKey sk) { Asm a { mcg->code.main() }; // Emit the checks for debugger attach - emitTLSLoad(a, ThreadInfo::s_threadInfo, reg::rAsm); - a. loadb (reg::rAsm[dbgOff], rbyte(reg::rAsm)); - a. testb ((int8_t)0xff, rbyte(reg::rAsm)); + auto rtmp = rAsm; + emitTLSLoad(a, ThreadInfo::s_threadInfo, rtmp); + a. loadb (rtmp[dbgOff], rbyte(rtmp)); + a. testb ((int8_t)0xff, rbyte(rtmp)); // Branch to a special REQ_INTERPRET if attached auto const fallback = diff --git a/hphp/runtime/vm/jit/extra-data.h b/hphp/runtime/vm/jit/extra-data.h index 7bd4aadc5e5..fdf4fe07785 100644 --- a/hphp/runtime/vm/jit/extra-data.h +++ b/hphp/runtime/vm/jit/extra-data.h @@ -744,7 +744,7 @@ struct RawMemData : IRExtraData { # undef RAW_TYPE struct Info { - const int64_t offset; + const int offset; const int size; const JIT::Type type; }; diff --git a/hphp/runtime/vm/jit/func-prologues-x64.cpp b/hphp/runtime/vm/jit/func-prologues-x64.cpp index ff0167ab767..c7bf6630e09 100644 --- a/hphp/runtime/vm/jit/func-prologues-x64.cpp +++ b/hphp/runtime/vm/jit/func-prologues-x64.cpp @@ -41,7 +41,7 @@ void emitStackCheck(X64Assembler& a, int funcDepth, Offset pc) { using namespace reg; funcDepth += kStackCheckPadding * sizeof(Cell); - uint64_t stackMask = cellsToBytes(RuntimeOption::EvalVMStackElms) - 1; + int stackMask = cellsToBytes(RuntimeOption::EvalVMStackElms) - 1; a. movq (rVmSp, rAsm); // copy to destroy a. andq (stackMask, rAsm); a. subq (funcDepth + Stack::sSurprisePageSize, rAsm); @@ -68,12 +68,13 @@ TCA emitFuncGuard(X64Assembler& a, const Func* func) { const int kAlign = kX64CacheLineSize; const int kAlignMask = kAlign - 1; + auto funcImm = Immed64(func); int loBits = uintptr_t(a.frontier()) & kAlignMask; int delta, size; // Ensure the immediate is safely smashable // the immediate must not cross a qword boundary, - if (!deltaFits((intptr_t)func, sz::dword)) { + if (!funcImm.fits(sz::dword)) { size = 8; delta = loBits + kFuncMovImm; } else { @@ -87,7 +88,7 @@ TCA emitFuncGuard(X64Assembler& a, const Func* func) { } TCA aStart DEBUG_ONLY = a.frontier(); - if (!deltaFits((intptr_t)func, sz::dword)) { + if (!funcImm.fits(sz::dword)) { a. loadq (rStashedAR[AROFF(m_func)], rax); /* Although func doesnt fit in a signed 32-bit immediate, it may still @@ -100,7 +101,7 @@ TCA emitFuncGuard(X64Assembler& a, const Func* func) { ((uint64_t*)a.frontier())[-1] = uintptr_t(func); a. cmpq (rax, rdx); } else { - a. cmpq (func, rStashedAR[AROFF(m_func)]); + a. cmpq (funcImm.l(), rStashedAR[AROFF(m_func)]); } a. jnz (tx->uniqueStubs.funcPrologueRedispatch); @@ -164,7 +165,7 @@ SrcKey emitPrologueWork(Func* func, int nPassed) { // This should be an unusual case, so optimize for code density // rather than execution speed; i.e., don't unroll the loop. TCA loopTop = a.frontier(); - a. subq (sizeof(Cell), rVmSp); + a. subq (int(sizeof(Cell)), rVmSp); a. incl (eax); emitStoreUninitNull(a, 0, rVmSp); a. cmpl (numParams, eax); @@ -252,7 +253,7 @@ SrcKey emitPrologueWork(Func* func, int nPassed) { // } while(++loopReg != loopEnd); emitStoreTVType(a, edx, rVmFp[loopReg]); - a. addq (sizeof(Cell), loopReg); + a. addq (int(sizeof(Cell)), loopReg); a. cmpq (loopEnd, loopReg); a. jcc8 (CC_NE, topOfLoop); } else { @@ -427,7 +428,8 @@ SrcKey emitMagicFuncPrologue(Func* func, uint32_t nPassed, TCA& start) { a. storeq (rInvName, strTV[TVOFF(m_data)]); emitStoreTVType(a, KindOfArray, arrayTV[TVOFF(m_type)]); if (nPassed == 0) { - a. storeq (HphpArray::GetStaticEmptyArray(), arrayTV[TVOFF(m_data)]); + emitImmStoreq(a, HphpArray::GetStaticEmptyArray(), + arrayTV[TVOFF(m_data)]); } else { a. storeq (rax, arrayTV[TVOFF(m_data)]); } diff --git a/hphp/runtime/vm/jit/ir-translator.cpp b/hphp/runtime/vm/jit/ir-translator.cpp index b1194db067f..7afda7c68e3 100644 --- a/hphp/runtime/vm/jit/ir-translator.cpp +++ b/hphp/runtime/vm/jit/ir-translator.cpp @@ -59,9 +59,6 @@ static const bool debug = true; static const bool debug = false; #endif -#define TVOFF(nm) offsetof(TypedValue, nm) -#define AROFF(nm) offsetof(ActRec, nm) - #define HHIR_EMIT(op, ...) \ do { \ m_hhbcTrans.emit ## op(__VA_ARGS__); \ diff --git a/hphp/runtime/vm/jit/phys-reg.cpp b/hphp/runtime/vm/jit/phys-reg.cpp index 447d4e22ade..e4fe834025d 100644 --- a/hphp/runtime/vm/jit/phys-reg.cpp +++ b/hphp/runtime/vm/jit/phys-reg.cpp @@ -79,7 +79,7 @@ int PhysRegSaverParity::rspTotalAdjustmentRegs() const { return m_regs.size() + m_adjust / sizeof(int64_t); } -void PhysRegSaverParity::bytesPushed(int64_t bytes) { +void PhysRegSaverParity::bytesPushed(int bytes) { m_adjust += bytes; } diff --git a/hphp/runtime/vm/jit/phys-reg.h b/hphp/runtime/vm/jit/phys-reg.h index 34c0e094e66..686af89cf50 100644 --- a/hphp/runtime/vm/jit/phys-reg.h +++ b/hphp/runtime/vm/jit/phys-reg.h @@ -397,12 +397,12 @@ struct PhysRegSaverParity { int rspAdjustment() const; int rspTotalAdjustmentRegs() const; - void bytesPushed(int64_t bytes); + void bytesPushed(int bytes); private: X64Assembler& m_as; RegSet m_regs; - int64_t m_adjust; + int m_adjust; }; struct PhysRegSaverStub : public PhysRegSaverParity { diff --git a/hphp/runtime/vm/jit/reg-alloc.cpp b/hphp/runtime/vm/jit/reg-alloc.cpp index b418e8a5361..3663ace3c83 100644 --- a/hphp/runtime/vm/jit/reg-alloc.cpp +++ b/hphp/runtime/vm/jit/reg-alloc.cpp @@ -223,7 +223,10 @@ Constraint dstConstraint(const IRInstruction& inst, unsigned i) { namespace X64 { -bool okStore(int64_t c) { return isI32(c); } +// okStore is true if cgStore can take c as an immediate without +// using any scratch registers. +bool okStore(int64_t c) { return true; } + bool okCmp(int64_t c) { return isI32(c); } // return true if CodeGenerator supports this operand as an @@ -349,6 +352,18 @@ bool mayUseConst(const IRInstruction& inst, unsigned i) { case CheckPackedArrayBounds: if (i == 1) return okCmp(cint); // idx break; + case CheckBounds: + if (i == 0) { // idx + return !inst.src(1)->isConst() && okCmp(cint) && cint >= 0; + } + if (i == 1) { // size + return !inst.src(0)->isConst() && okCmp(cint) && cint >= 0; + } + break; + case VerifyParamCls: + case VerifyRetCls: + if (i == 1) return okCmp(cint); // constraint class ptr + break; default: break; } diff --git a/hphp/runtime/vm/jit/service-requests-x64.cpp b/hphp/runtime/vm/jit/service-requests-x64.cpp index cc0cb1dcb8f..58b9149704e 100644 --- a/hphp/runtime/vm/jit/service-requests-x64.cpp +++ b/hphp/runtime/vm/jit/service-requests-x64.cpp @@ -55,7 +55,7 @@ StoreImmPatcher::StoreImmPatcher(CodeBlock& cb, uint64_t initial, X64Assembler as { cb }; m_is32 = deltaFits(initial, sz::dword); if (m_is32) { - as.storeq(initial, r64(base)[offset]); + as.storeq(int32_t(initial), r64(base)[offset]); m_addr = cb.frontier() - 4; } else { as.movq(initial, r64(reg)); @@ -248,9 +248,9 @@ emitServiceReqWork(CodeBlock& cb, TCA start, bool persist, SRFlags flags, } emitEagerVMRegSave(as, RegSaveFlags::SaveFP); if (persist) { - as. emitImmReg(0, JIT::reg::rAsm); + as. emitImmReg(0, JIT::X64::rAsm); } else { - as. emitImmReg((uint64_t)start, JIT::reg::rAsm); + as. emitImmReg((uint64_t)start, JIT::X64::rAsm); } TRACE(3, ")\n"); as. emitImmReg(req, JIT::reg::rdi); @@ -326,8 +326,8 @@ int32_t emitBindCall(CodeBlock& mainCode, CodeBlock& stubsCode, Asm a { mainCode }; if (debug) { - a. storeq (kUninitializedRIP, - rVmSp[cellsToBytes(numArgs) + AROFF(m_savedRip)]); + auto off = cellsToBytes(numArgs) + AROFF(m_savedRip); + emitImmStoreq(a, kUninitializedRIP, rVmSp[off]); } // Stash callee's rVmFp into rStashedAR for the callee's prologue emitLea(a, rVmSp[cellsToBytes(numArgs)], rStashedAR); diff --git a/hphp/runtime/vm/jit/translator-inline.h b/hphp/runtime/vm/jit/translator-inline.h index cbf403e347e..7d4c8ab5a23 100644 --- a/hphp/runtime/vm/jit/translator-inline.h +++ b/hphp/runtime/vm/jit/translator-inline.h @@ -22,9 +22,6 @@ #include #include "hphp/runtime/base/execution-context.h" -#define TVOFF(nm) offsetof(TypedValue, nm) -#define AROFF(nm) offsetof(ActRec, nm) - /* * Because of a circular dependence with ExecutionContext, these * translation-related helpers cannot live in translator.h. @@ -62,11 +59,11 @@ inline bool isNativeImplCall(const Func* funcd, int numArgs) { return funcd && funcd->methInfo() && numArgs == funcd->numParams(); } -inline ptrdiff_t cellsToBytes(int nCells) { - return nCells * sizeof(Cell); +inline int cellsToBytes(int nCells) { + return safe_cast(nCells * ssize_t(sizeof(Cell))); } -inline int64_t localOffset(int64_t locId) { +inline int localOffset(int locId) { return -cellsToBytes(locId + 1); } diff --git a/hphp/runtime/vm/jit/type.h b/hphp/runtime/vm/jit/type.h index e349e67367c..3e20445e0bb 100644 --- a/hphp/runtime/vm/jit/type.h +++ b/hphp/runtime/vm/jit/type.h @@ -839,8 +839,8 @@ struct TypeConstraint { Type assertedType; }; -const size_t kTypeWordOffset = (offsetof(TypedValue, m_type) % 8); -const size_t kTypeShiftBits = kTypeWordOffset * CHAR_BIT; +const int kTypeWordOffset = offsetof(TypedValue, m_type) % 8; +const int kTypeShiftBits = kTypeWordOffset * CHAR_BIT; // left shift an immediate DataType, for type, to the correct position // within one of the registers used to pass a TypedValue by value. diff --git a/hphp/runtime/vm/jit/unique-stubs-x64.cpp b/hphp/runtime/vm/jit/unique-stubs-x64.cpp index b7ac103395e..b2c5f4b1839 100644 --- a/hphp/runtime/vm/jit/unique-stubs-x64.cpp +++ b/hphp/runtime/vm/jit/unique-stubs-x64.cpp @@ -172,6 +172,7 @@ void emitFreeLocalsHelpers(UniqueStubs& uniqueStubs) { auto const rFinished = r15; auto const rType = esi; auto const rData = rdi; + int const tvSize = sizeof(TypedValue); Asm a { mcg->code.main() }; moveToAlign(mcg->code.main(), kNonFallthroughAlign); @@ -207,7 +208,7 @@ asm_label(a, doRelease); // kNumFreeLocalsHelpers. asm_label(a, loopHead); emitDecLocal(); - a. addq (sizeof(TypedValue), rIter); + a. addq (tvSize, rIter); a. cmpq (rIter, rFinished); a. jnz8 (loopHead); @@ -215,11 +216,11 @@ asm_label(a, loopHead); uniqueStubs.freeLocalsHelpers[kNumFreeLocalsHelpers - i - 1] = a.frontier(); emitDecLocal(); if (i != kNumFreeLocalsHelpers - 1) { - a.addq (sizeof(TypedValue), rIter); + a.addq (tvSize, rIter); } } - a. addq (AROFF(m_r) + sizeof(TypedValue), rVmSp); + a. addq (AROFF(m_r) + tvSize, rVmSp); a. ret (); uniqueStubs.add("freeLocalsHelpers", uniqueStubs.freeManyLocalsHelper); diff --git a/hphp/runtime/vm/native.h b/hphp/runtime/vm/native.h index 0821db9dcc8..5dc1d5748c4 100644 --- a/hphp/runtime/vm/native.h +++ b/hphp/runtime/vm/native.h @@ -144,7 +144,7 @@ inline int maxFCallBuiltinArgs() { if (UNLIKELY(RuntimeOption::EvalSimulateARM)) { return kMaxFCallBuiltinArgsARM; } - return 5; // kMaxBuiltinArgsNoDouble; + return kMaxBuiltinArgsNoDouble; #endif } diff --git a/hphp/util/asm-x64.h b/hphp/util/asm-x64.h index 29174bba425..f5f68f2b277 100644 --- a/hphp/util/asm-x64.h +++ b/hphp/util/asm-x64.h @@ -460,9 +460,6 @@ namespace reg { constexpr RegXMM xmm14(14); constexpr RegXMM xmm15(15); - // rAsm is the symbolic name for a reg that is reserved for the assembler - constexpr Reg64 rAsm(r10); - #define X(x) if (r == x) return "%"#x inline const char* regname(Reg64 r) { X(rax); X(rbx); X(rcx); X(rdx); X(rsp); X(rbp); X(rsi); X(rdi); @@ -830,39 +827,24 @@ public: /* * For when we a have a memory operand and the operand size is * 64-bits, only a 32-bit (sign-extended) immediate is supported. - * If the immediate is too big, we'll move it into rAsm first. */ #define IMM64_STORE_OP(name, instr) \ void name##q(Immed i, MemoryRef m) { \ - if (i.fits(sz::dword)) { \ - return instrIM(instr, i, m); \ - } \ - movq (i, reg::rAsm); \ - name##q(reg::rAsm, m); \ + return instrIM(instr, i, m); \ } \ \ void name##q(Immed i, IndexedMemoryRef m) { \ - if (i.fits(sz::dword)) { \ - return instrIM(instr, i, m); \ - } \ - movq (i, reg::rAsm); \ - name##q(reg::rAsm, m); \ + return instrIM(instr, i, m); \ } /* * For instructions other than movq, even when the operand size is - * 64 bits only a 32-bit (sign-extended) immediate is supported. We - * provide foo##q instructions that may emit multiple x64 - * instructions (smashing rAsm) if the immediate does not - * actually fit in a long. + * 64 bits only a 32-bit (sign-extended) immediate is supported. */ #define IMM64R_OP(name, instr) \ void name##q(Immed imm, Reg64 r) { \ - if (imm.fits(sz::dword)) { \ - return instrIR(instr, imm, r); \ - } \ - movq (imm, reg::rAsm); \ - name##q(reg::rAsm, r); \ + always_assert(imm.fits(sz::dword)); \ + return instrIR(instr, imm, r); \ } #define FULL_OP(name, instr) \ @@ -899,7 +881,7 @@ public: #undef BYTE_REG_OP // 64-bit immediates work with mov to a register. - void movq(Immed imm, Reg64 r) { instrIR(instr_mov, imm, r); } + void movq(Immed64 imm, Reg64 r) { instrIR(instr_mov, imm, r); } // movzbx is a special snowflake. We don't have movzbq because it behaves // exactly the same as movzbl but takes an extra byte. @@ -929,11 +911,6 @@ public: void imul(Reg64 r1, Reg64 r2) { instrRR(instr_imul, r1, r2); } - void imul(Immed im, Reg64 r1) { - movq(im, reg::rAsm); - imul(reg::rAsm, r1); - } - void push(Reg64 r) { instrR(instr_push, r); } void pushl(Reg32 r) { instrR(instr_push, r); } void pop (Reg64 r) { instrR(instr_pop, r); } @@ -985,13 +962,14 @@ public: void lddqu (IndexedMemoryRef m, RegXMM x) { instrMR(instr_lddqu, m, x); } void unpcklpd(RegXMM s, RegXMM d) { instrRR(instr_unpcklpd, d, s); } - void shlq (Immed i, Reg64 r) { instrIR(instr_shl, i.b(), r); } - void shrq (Immed i, Reg64 r) { instrIR(instr_shr, i.b(), r); } - void sarq (Immed i, Reg64 r) { instrIR(instr_sar, i.b(), r); } - void shll (Immed i, Reg32 r) { instrIR(instr_shl, i.b(), r); } - void shrl (Immed i, Reg32 r) { instrIR(instr_shr, i.b(), r); } - void shlw (Immed i, Reg16 r) { instrIR(instr_shl, i.b(), r); } - void shrw (Immed i, Reg16 r) { instrIR(instr_shr, i.b(), r); } + void rorq (Immed i, Reg64 r) { instrIR(instr_ror, i, r); } + void shlq (Immed i, Reg64 r) { instrIR(instr_shl, i, r); } + void shrq (Immed i, Reg64 r) { instrIR(instr_shr, i, r); } + void sarq (Immed i, Reg64 r) { instrIR(instr_sar, i, r); } + void shll (Immed i, Reg32 r) { instrIR(instr_shl, i, r); } + void shrl (Immed i, Reg32 r) { instrIR(instr_shr, i, r); } + void shlw (Immed i, Reg16 r) { instrIR(instr_shl, i, r); } + void shrw (Immed i, Reg16 r) { instrIR(instr_shr, i, r); } void shlq (Reg64 r) { instrR(instr_shl, r); } void sarq (Reg64 r) { instrR(instr_sar, r); } @@ -1020,25 +998,13 @@ public: void jmp8(CodeAddress dest) { emitJ8(instr_jmp, ssize_t(dest)); } - // May smash rAsm. void jmp(CodeAddress dest) { - always_assert(dest); - if (!jmpDeltaFits(dest)) { - movq (dest, reg::rAsm); - jmp (reg::rAsm); - return; - } + always_assert(dest && jmpDeltaFits(dest)); emitJ32(instr_jmp, ssize_t(dest)); } - // May smash rAsm. void call(CodeAddress dest) { - always_assert(dest); - if (!jmpDeltaFits(dest)) { - movq (dest, reg::rAsm); - call (reg::rAsm); - return; - } + always_assert(dest && jmpDeltaFits(dest)); emitJ32(instr_call, ssize_t(dest)); } @@ -1163,7 +1129,7 @@ public: * (E.g. combine common idioms or patterns, smash code, etc.) */ - void emitImmReg(Immed imm, Reg64 dest) { + void emitImmReg(Immed64 imm, Reg64 dest) { if (imm.q() == 0) { // Zeros the top bits also. xorl (r32(rn(dest)), r32(rn(dest))); @@ -2111,6 +2077,9 @@ private: Reg64 r) { emitMR(op, URIP(m), rn(r), sz::qword, true); } + void instrIR(X64Instr op, Immed64 i, Reg64 r) { + emitIR(op, rn(r), i.q()); + } void instrIR(X64Instr op, Immed i, Reg64 r) { emitIR(op, rn(r), i.q()); } diff --git a/hphp/util/immed.h b/hphp/util/immed.h index 1d9b31dce05..8915e69651c 100644 --- a/hphp/util/immed.h +++ b/hphp/util/immed.h @@ -45,9 +45,13 @@ inline bool magFits(uint64_t val, int s) { } /* - * Immediate wrapper for the assembler. + * Immediate wrappers for the assembler. * - * This wrapper picks up whether the immediate argument was an integer + * Immed only allows 32-bit signed values. Unsigned-32bit values and + * larger values will not safely fit, we want the caller to deal with + * it or explicitly downcast to int32_t. + * + * The Immed64 wrapper picks up whether the immediate argument was an integer * or a pointer type, so we don't have to cast pointers at callsites. * * Immediates are always treated as sign-extended values, but it's @@ -55,19 +59,10 @@ inline bool magFits(uint64_t val, int s) { * implicit implementation-defined conversion. */ struct Immed { - template - /* implicit */ Immed(T i, - typename std::enable_if< - std::is_integral::value || - std::is_enum::value - >::type* = 0) - : m_int(i) - {} - - template - /* implicit */ Immed(T* p) - : m_int(reinterpret_cast(p)) - {} + /* implicit */ Immed(int32_t i) : m_int(i) {} + /* implicit */ Immed(uint32_t i) = delete; + /* implicit */ Immed(int64_t i) = delete; + /* implicit */ Immed(uint64_t i) = delete; int64_t q() const { return m_int; } int32_t l() const { return safe_cast(m_int); } @@ -77,7 +72,33 @@ struct Immed { bool fits(int sz) const { return deltaFits(m_int, sz); } private: - intptr_t m_int; + int32_t m_int; +}; + +struct Immed64 { + template + /* implicit */ Immed64(T i, + typename std::enable_if< + std::is_integral::value || + std::is_enum::value + >::type* = 0) + : m_long(i) + {} + + template + /* implicit */ Immed64(T* p) + : m_long(reinterpret_cast(p)) + {} + + int64_t q() const { return m_long; } + int32_t l() const { return safe_cast(m_long); } + int16_t w() const { return safe_cast(m_long); } + int8_t b() const { return safe_cast(m_long); } + + bool fits(int sz) const { return deltaFits(m_long, sz); } + +private: + int64_t m_long; }; }} diff --git a/hphp/util/test/asm.cpp b/hphp/util/test/asm.cpp index cfa01180db0..94e67ad926c 100644 --- a/hphp/util/test/asm.cpp +++ b/hphp/util/test/asm.cpp @@ -687,7 +687,8 @@ TEST(Asm, SimpleLabelTest) { asm_label(a, loop); a. movq (r15, rdi); - a. call (CodeAddress(static_cast(loopCallee))); + a. movq (CodeAddress(static_cast(loopCallee)), r10); + a. call (r10); a. incl (ebx); a. cmpl (ebx, r12d); a. jne (loop); -- 2.11.4.GIT