2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
17 #include "hphp/runtime/vm/jit/codegen.h"
22 #include "folly/ScopeGuard.h"
23 #include "folly/Format.h"
24 #include "hphp/util/trace.h"
25 #include "hphp/util/util.h"
27 #include "hphp/runtime/base/array/hphp_array.h"
28 #include "hphp/runtime/base/comparisons.h"
29 #include "hphp/runtime/base/complex_types.h"
30 #include "hphp/runtime/base/runtime_option.h"
31 #include "hphp/runtime/base/string_data.h"
32 #include "hphp/runtime/base/types.h"
33 #include "hphp/runtime/ext/ext_closure.h"
34 #include "hphp/runtime/ext/ext_continuation.h"
35 #include "hphp/runtime/ext/ext_collections.h"
36 #include "hphp/runtime/vm/bytecode.h"
37 #include "hphp/runtime/vm/runtime.h"
38 #include "hphp/runtime/base/stats.h"
39 #include "hphp/runtime/vm/jit/targetcache.h"
40 #include "hphp/runtime/vm/jit/translator-inline.h"
41 #include "hphp/runtime/vm/jit/translator-x64.h"
42 #include "hphp/runtime/vm/jit/translator-x64-internal.h"
43 #include "hphp/runtime/vm/jit/translator.h"
44 #include "hphp/runtime/vm/jit/types.h"
45 #include "hphp/runtime/vm/jit/x64-util.h"
46 #include "hphp/runtime/vm/jit/ir.h"
47 #include "hphp/runtime/vm/jit/linearscan.h"
48 #include "hphp/runtime/vm/jit/nativecalls.h"
49 #include "hphp/runtime/vm/jit/print.h"
50 #include "hphp/runtime/vm/jit/layout.h"
52 using HPHP::Transl::TCA
;
53 using namespace HPHP::Transl::TargetCache
;
60 //////////////////////////////////////////////////////////////////////
63 using namespace Transl::reg
;
68 * It's not normally ok to directly use tracelet abi registers in
69 * codegen, unless you're directly dealing with an instruction that
70 * does near-end-of-tracelet glue. (Or also we sometimes use them
71 * just for some static_assertions relating to calls to helpers from
72 * tx64 that hardcode these registers.)
77 const size_t kTypeWordOffset
= (offsetof(TypedValue
, m_type
) % 8);
78 const size_t kTypeShiftBits
= kTypeWordOffset
* CHAR_BIT
;
80 // left shift an immediate DataType, for type, to the correct position
81 // within one of the registers used to pass a TypedValue by value.
82 uint64_t toDataTypeForCall(Type type
) {
83 return uint64_t(type
.toDataType()) << kTypeShiftBits
;
86 void cgPunt(const char* file
, int line
, const char* func
, uint32_t bcOff
,
88 if (dumpIREnabled()) {
89 HPHP::Trace::trace("--------- CG_PUNT %s %d %s bcOff: %d \n",
90 file
, line
, func
, bcOff
);
92 throw FailedCodeGen(file
, line
, func
, bcOff
, vmFunc
);
95 #define CG_PUNT(instr) \
96 cgPunt(__FILE__, __LINE__, #instr, m_curInst->marker().bcOff, curFunc())
104 enum class Kind
{ Move
, Xchg
};
106 MoveInfo(Kind kind
, int reg1
, int reg2
):
107 m_kind(kind
), m_reg1(reg1
), m_reg2(reg2
) {}
110 PhysReg m_reg1
, m_reg2
;
114 static bool cycleHasXMMReg(const CycleInfo
& cycle
,
115 const int (&moves
)[N
]) {
116 int first
= cycle
.node
;
119 if (PhysReg(node
).isXMM()) return true;
121 } while (node
!= first
);
126 void doRegMoves(int (&moves
)[N
], int rTmp
,
127 std::vector
<MoveInfo
>& howTo
) {
128 assert(howTo
.empty());
132 // Iterate over the nodes filling in outDegree[] and cycles[] as we go
135 for (int node
= 0; node
< N
; ++node
) {
136 // If a node's source is itself, its a nop
137 if (moves
[node
] == node
) moves
[node
] = -1;
138 if (node
== rTmp
&& moves
[node
] >= 0) {
139 // ERROR: rTmp cannot be referenced in moves[].
146 for (int startNode
= 0; startNode
< N
; ++startNode
) {
147 // If startNode has not been visited yet, begin walking
148 // a path from start node
149 if (index
[startNode
] < 0) {
150 int node
= startNode
;
152 index
[node
] = nextIndex
++;
153 if (moves
[node
] >= 0) {
154 int nextNode
= moves
[node
];
155 ++outDegree
[nextNode
];
156 if (index
[nextNode
] < 0) {
157 // If there is an edge from v to nextNode and nextNode has not been
158 // visited, extend the current path to include nextNode and recurse
162 // There is an edge from v to nextNode but nextNode has already been
163 // visited, check if nextNode is on the current path
164 if (index
[nextNode
] >= index
[startNode
]) {
165 // nextNode is on the current path so we've found a cycle
166 int length
= nextIndex
- index
[nextNode
];
167 CycleInfo ci
= { nextNode
, length
};
168 cycles
[numCycles
] = ci
;
175 // Handle all moves that aren't part of a cycle
179 for (int node
= 0; node
< N
; ++node
) {
180 if (outDegree
[node
] == 0) {
185 for (int i
= 0; i
< qBack
; ++i
) {
187 if (moves
[node
] >= 0) {
188 int nextNode
= moves
[node
];
189 howTo
.push_back(MoveInfo(MoveInfo::Kind::Move
, nextNode
, node
));
190 --outDegree
[nextNode
];
191 if (outDegree
[nextNode
] == 0) {
198 // Deal with any cycles we encountered
199 for (int i
= 0; i
< numCycles
; ++i
) {
200 // can't use xchg if one of the registers is XMM
201 bool hasXMMReg
= cycleHasXMMReg(cycles
[i
], moves
);
202 if (cycles
[i
].length
== 2 && !hasXMMReg
) {
203 int v
= cycles
[i
].node
;
205 howTo
.push_back(MoveInfo(MoveInfo::Kind::Xchg
, w
, v
));
206 } else if (cycles
[i
].length
== 3 && !hasXMMReg
) {
207 int v
= cycles
[i
].node
;
209 howTo
.push_back(MoveInfo(MoveInfo::Kind::Xchg
, w
, v
));
211 howTo
.push_back(MoveInfo(MoveInfo::Kind::Xchg
, x
, w
));
213 int v
= cycles
[i
].node
;
214 howTo
.push_back(MoveInfo(MoveInfo::Kind::Move
, v
, rTmp
));
218 howTo
.push_back(MoveInfo(MoveInfo::Kind::Move
, x
, w
));
222 howTo
.push_back(MoveInfo(MoveInfo::Kind::Move
, rTmp
, w
));
227 const char* getContextName(Class
* ctx
) {
228 return ctx
? ctx
->name()->data() : ":anonymous:";
231 } // unnamed namespace
233 //////////////////////////////////////////////////////////////////////
235 ArgDesc::ArgDesc(SSATmp
* tmp
, const RegisterInfo
& info
, bool val
)
236 : m_imm(-1), m_zeroExtend(false), m_done(false) {
237 if (tmp
->type() == Type::None
) {
242 if (tmp
->inst()->op() == DefConst
) {
243 m_srcReg
= InvalidReg
;
245 m_imm
= tmp
->getValBits();
247 m_imm
= toDataTypeForCall(tmp
->type());
252 if (tmp
->type().isNull()) {
253 m_srcReg
= InvalidReg
;
257 m_imm
= toDataTypeForCall(tmp
->type());
262 if (val
|| tmp
->numNeededRegs() > 1) {
263 auto reg
= info
.reg(val
? 0 : 1);
264 assert(reg
!= InvalidReg
);
267 // If val is false then we're passing tmp's type. TypeReg lets
268 // CodeGenerator know that the value might require some massaging
269 // to be in the right format for the call.
270 m_kind
= val
? Kind::Reg
: Kind::TypeReg
;
271 // zero extend any boolean value that we pass to the helper in case
272 // the helper expects it (e.g., as TypedValue)
273 if (val
&& tmp
->isA(Type::Bool
)) m_zeroExtend
= true;
277 m_srcReg
= InvalidReg
;
278 m_imm
= toDataTypeForCall(tmp
->type());
282 const Func
* CodeGenerator::curFunc() const {
283 assert(m_curInst
->marker().valid());
284 return m_curInst
->marker().func
;
288 * Select a scratch register to use in the given instruction, prefering the
289 * lower registers which don't require a REX prefix. The selected register
290 * must not be any of the instructions inputs or outputs, and neither a register
291 * that is alive across this instruction.
293 PhysReg
CodeGenerator::selectScratchReg(IRInstruction
* inst
) {
294 static const RegSet kLowGPRegs
= RegSet()
301 RegSet liveRegs
= m_state
.liveRegs
[inst
];
302 for (const auto& tmp
: inst
->srcs()) {
303 liveRegs
|= m_regs
[tmp
].regs();
305 for (const auto& tmp
: inst
->dsts()) {
306 liveRegs
|= m_regs
[tmp
].regs();
309 if ((kLowGPRegs
- liveRegs
).findFirst(selectedReg
)) {
315 Address
CodeGenerator::cgInst(IRInstruction
* inst
) {
316 Opcode opc
= inst
->op();
317 auto const start
= m_as
.code
.frontier
;
318 m_rScratch
= selectScratchReg(inst
);
319 if (inst
->taken() && inst
->taken()->trace()->isCatch()) {
320 m_state
.catchTrace
= inst
->taken()->trace();
322 m_state
.catchTrace
= nullptr;
326 #define O(name, dsts, srcs, flags) \
327 case name: FTRACE(7, "cg" #name "\n"); \
329 return m_as.code.frontier == start ? nullptr : start;
339 #define NOOP_OPCODE(opcode) \
340 void CodeGenerator::cg##opcode(IRInstruction*) {}
342 #define CALL_OPCODE(opcode) \
343 void CodeGenerator::cg##opcode(IRInstruction* i) { cgCallNative(i); }
345 #define CALL_STK_OPCODE(opcode) \
346 CALL_OPCODE(opcode) \
347 CALL_OPCODE(opcode ## Stk)
349 NOOP_OPCODE(DefConst
)
352 NOOP_OPCODE(AssertLoc
)
353 NOOP_OPCODE(OverrideLoc
)
354 NOOP_OPCODE(SmashLocals
)
355 NOOP_OPCODE(AssertStk
)
356 NOOP_OPCODE(AssertStkVal
)
358 NOOP_OPCODE(DefLabel
)
359 NOOP_OPCODE(ExceptionBarrier
)
361 CALL_OPCODE(AddElemStrKey
)
362 CALL_OPCODE(AddElemIntKey
)
363 CALL_OPCODE(AddNewElem
)
364 CALL_OPCODE(ArrayAdd
)
367 CALL_OPCODE(ConvBoolToArr
);
368 CALL_OPCODE(ConvDblToArr
);
369 CALL_OPCODE(ConvIntToArr
);
370 CALL_OPCODE(ConvObjToArr
);
371 CALL_OPCODE(ConvStrToArr
);
372 CALL_OPCODE(ConvCellToArr
);
374 CALL_OPCODE(ConvArrToBool
);
375 CALL_OPCODE(ConvStrToBool
);
376 CALL_OPCODE(ConvCellToBool
);
378 CALL_OPCODE(ConvArrToDbl
);
379 CALL_OPCODE(ConvObjToDbl
);
380 CALL_OPCODE(ConvStrToDbl
);
381 CALL_OPCODE(ConvCellToDbl
);
383 CALL_OPCODE(ConvArrToInt
);
384 CALL_OPCODE(ConvDblToInt
);
385 CALL_OPCODE(ConvObjToInt
);
386 CALL_OPCODE(ConvStrToInt
);
387 CALL_OPCODE(ConvCellToInt
);
389 CALL_OPCODE(ConvCellToObj
);
391 CALL_OPCODE(ConvDblToStr
);
392 CALL_OPCODE(ConvIntToStr
);
393 CALL_OPCODE(ConvObjToStr
);
394 CALL_OPCODE(ConvCellToStr
);
396 CALL_OPCODE(CreateContFunc
)
397 CALL_OPCODE(CreateContMeth
)
398 CALL_OPCODE(NewArray
)
399 CALL_OPCODE(NewTuple
)
400 CALL_OPCODE(AllocObj
)
401 CALL_OPCODE(LdClsCtor
);
402 CALL_OPCODE(CreateCl
)
403 CALL_OPCODE(PrintStr
)
404 CALL_OPCODE(PrintInt
)
405 CALL_OPCODE(PrintBool
)
406 CALL_OPCODE(DbgAssertPtr
)
407 CALL_OPCODE(LdSwitchDblIndex
)
408 CALL_OPCODE(LdSwitchStrIndex
)
409 CALL_OPCODE(LdSwitchObjIndex
)
410 CALL_OPCODE(VerifyParamCallable
)
411 CALL_OPCODE(VerifyParamFail
)
412 CALL_OPCODE(RaiseUninitLoc
)
413 CALL_OPCODE(WarnNonObjProp
)
414 CALL_OPCODE(ThrowNonObjProp
)
415 CALL_OPCODE(RaiseUndefProp
)
416 CALL_OPCODE(RaiseError
)
417 CALL_OPCODE(RaiseWarning
)
418 CALL_OPCODE(IncStatGrouped
)
419 CALL_OPCODE(StaticLocInit
)
420 CALL_OPCODE(StaticLocInitCached
)
422 CALL_OPCODE(ArrayIdx
)
424 // Vector instruction helpers
427 CALL_STK_OPCODE(PropDX
)
428 CALL_OPCODE(CGetProp
)
429 CALL_STK_OPCODE(VGetProp
)
430 CALL_STK_OPCODE(BindProp
)
431 CALL_STK_OPCODE(SetProp
)
432 CALL_OPCODE(UnsetProp
)
433 CALL_STK_OPCODE(SetOpProp
)
434 CALL_STK_OPCODE(IncDecProp
)
435 CALL_OPCODE(EmptyProp
)
436 CALL_OPCODE(IssetProp
)
438 CALL_STK_OPCODE(ElemDX
)
439 CALL_STK_OPCODE(ElemUX
)
440 CALL_OPCODE(ArrayGet
)
441 CALL_OPCODE(VectorGet
)
444 CALL_OPCODE(StableMapGet
)
445 CALL_OPCODE(CGetElem
)
446 CALL_STK_OPCODE(VGetElem
)
447 CALL_STK_OPCODE(BindElem
)
448 CALL_STK_OPCODE(SetWithRefElem
)
449 CALL_STK_OPCODE(SetWithRefNewElem
)
450 CALL_OPCODE(ArraySet
)
451 CALL_OPCODE(VectorSet
)
453 CALL_OPCODE(StableMapSet
)
454 CALL_OPCODE(ArraySetRef
)
455 CALL_STK_OPCODE(SetElem
)
456 CALL_STK_OPCODE(UnsetElem
)
457 CALL_STK_OPCODE(SetOpElem
)
458 CALL_STK_OPCODE(IncDecElem
)
459 CALL_STK_OPCODE(SetNewElem
)
460 CALL_STK_OPCODE(BindNewElem
)
461 CALL_OPCODE(ArrayIsset
)
462 CALL_OPCODE(VectorIsset
)
463 CALL_OPCODE(PairIsset
)
464 CALL_OPCODE(MapIsset
)
465 CALL_OPCODE(StableMapIsset
)
466 CALL_OPCODE(IssetElem
)
467 CALL_OPCODE(EmptyElem
)
471 // Thread chain of patch locations using the 4 byte space in each jmp/jcc
472 static void prependPatchAddr(CodegenState
& state
,
475 auto &patches
= state
.patches
;
476 ssize_t diff
= patches
[block
] ? (patchAddr
- (TCA
)patches
[block
]) : 0;
477 assert(deltaFits(diff
, sz::dword
));
478 *(int32_t*)(patchAddr
) = (int32_t)diff
;
479 patches
[block
] = patchAddr
;
482 static void emitFwdJmp(Asm
& a
, Block
* target
, CodegenState
& state
) {
483 if (auto addr
= state
.addresses
[target
]) {
484 return a
.jmpAuto(addr
);
487 // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
488 a
.jmp(a
.code
.frontier
);
489 TCA immPtr
= a
.code
.frontier
- 4;
490 prependPatchAddr(state
, target
, immPtr
);
493 void CodeGenerator::emitFwdJcc(Asm
& a
, ConditionCode cc
, Block
* target
) {
494 if (auto addr
= m_state
.addresses
[target
]) {
495 return a
.jccAuto(cc
, addr
);
498 // TODO(#2101926): it'd be nice to get 1-byte forward jumps here
499 a
.jcc(cc
, a
.code
.frontier
);
500 TCA immPtr
= a
.code
.frontier
- 4;
501 prependPatchAddr(m_state
, target
, immPtr
);
504 void CodeGenerator::emitFwdJcc(ConditionCode cc
, Block
* target
) {
505 emitFwdJcc(m_as
, cc
, target
);
508 void emitLoadImm(CodeGenerator::Asm
& as
, int64_t val
, PhysReg dstReg
) {
509 as
.emitImmReg(val
, dstReg
);
513 emitMovRegReg(CodeGenerator::Asm
& as
, PhysReg srcReg
, PhysReg dstReg
) {
514 assert(srcReg
!= InvalidReg
);
515 assert(dstReg
!= InvalidReg
);
517 if (srcReg
== dstReg
) return;
520 if (dstReg
.isGP()) { // GP => GP
521 as
.movq(srcReg
, dstReg
);
522 } else { // GP => XMM
523 // This generates a movq x86 instruction, which zero extends
524 // the 64-bit value in srcReg into a 128-bit XMM register
525 as
.mov_reg64_xmm(srcReg
, dstReg
);
528 if (dstReg
.isGP()) { // XMM => GP
529 as
.mov_xmm_reg64(srcReg
, dstReg
);
530 } else { // XMM => XMM
531 // This copies all 128 bits in XMM,
532 // thus avoiding partial register stalls
533 as
.movdqa(srcReg
, dstReg
);
538 void CodeGenerator::emitLoadImm(CodeGenerator::Asm
& as
, int64_t val
,
540 assert(dstReg
!= InvalidReg
);
542 as
.emitImmReg(val
, dstReg
);
544 assert(dstReg
.isXMM());
546 as
.pxor_xmm_xmm(dstReg
, dstReg
);
548 // Can't move immediate directly into XMM register, so use m_rScratch
549 as
.emitImmReg(val
, m_rScratch
);
550 emitMovRegReg(as
, m_rScratch
, dstReg
);
555 static void emitLea(CodeGenerator::Asm
& as
, MemoryRef mr
, PhysReg dst
) {
556 if (dst
== InvalidReg
) return;
557 if (mr
.r
.disp
== 0) {
558 emitMovRegReg(as
, mr
.r
.base
, dst
);
565 static void emitLoadReg(CodeGenerator::Asm
& as
, Mem mem
, PhysReg reg
) {
566 assert(reg
!= InvalidReg
);
575 static void emitStoreReg(CodeGenerator::Asm
& as
, PhysReg reg
, Mem mem
) {
576 assert(reg
!= InvalidReg
);
584 static void shuffle2(CodeGenerator::Asm
& a
,
585 PhysReg s0
, PhysReg s1
, PhysReg d0
, PhysReg d1
) {
587 if (d0
== s1
&& d1
!= InvalidReg
) {
592 a
. movq (s1
, d1
); // save s1 first; d1 != s0
596 if (d0
!= InvalidReg
) emitMovRegReg(a
, s0
, d0
); // d0 != s1
597 if (d1
!= InvalidReg
) emitMovRegReg(a
, s1
, d1
);
601 static void zeroExtendBool(X64Assembler
& as
, const RegisterInfo
& info
) {
602 auto reg
= info
.reg();
603 if (reg
!= InvalidReg
) {
604 // zero-extend the bool from a byte to a quad
605 // note: movzbl actually extends the value to 64 bits.
606 as
.movzbl(rbyte(reg
), r32(reg
));
610 static void zeroExtendIfBool(X64Assembler
& as
, const SSATmp
* src
,
611 const RegisterInfo
& info
) {
612 if (src
->isA(Type::Bool
)) {
613 zeroExtendBool(as
, info
);
617 static int64_t convIntToDouble(int64_t i
) {
627 * Returns a XMM register containing the value of SSATmp tmp,
628 * which can be either a bool, an int, or a double.
629 * If the value is already in a XMM register, simply returns it.
630 * Otherwise, the value is moved into rCgXMM, which is returned.
631 * If instructions to convert to a double at runtime are needed,
632 * they're emitted in 'as'.
634 PhysReg
CodeGenerator::prepXMMReg(const SSATmp
* tmp
,
636 const RegAllocInfo
& allocInfo
,
638 assert(tmp
->isA(Type::Bool
) || tmp
->isA(Type::Int
) || tmp
->isA(Type::Dbl
));
640 PhysReg reg
= allocInfo
[tmp
].reg();
642 // Case 1: tmp is already in a XMM register
643 if (reg
.isXMM()) return reg
;
645 // Case 2: tmp is in a GP register
646 if (reg
!= InvalidReg
) {
647 // Case 2.a: Dbl stored in GP reg
648 if (tmp
->isA(Type::Dbl
)) {
649 emitMovRegReg(as
, reg
, rCgXMM
);
652 // Case 2.b: Bool or Int stored in GP reg
653 assert(tmp
->isA(Type::Bool
) || tmp
->isA(Type::Int
));
654 zeroExtendIfBool(as
, tmp
, allocInfo
[tmp
]);
655 as
.pxor_xmm_xmm(rCgXMM
, rCgXMM
);
656 as
.cvtsi2sd_reg64_xmm(reg
, rCgXMM
);
660 // Case 3: tmp is a constant
661 assert(tmp
->isConst());
663 int64_t val
= tmp
->getValRawInt();
664 if (!tmp
->isA(Type::Dbl
)) {
665 assert(tmp
->isA(Type::Bool
| Type::Int
));
666 if (tmp
->isA(Type::Bool
)) val
= val
!= 0; // see task #2401790
667 val
= convIntToDouble(val
);
669 emitLoadImm(as
, val
, m_rScratch
);
670 emitMovRegReg(as
, m_rScratch
, rCgXMM
);
674 void CodeGenerator::doubleCmp(X64Assembler
& a
, RegXMM xmmReg0
, RegXMM xmmReg1
) {
675 a
. ucomisd_xmm_xmm(xmmReg0
, xmmReg1
);
678 // PF means the doubles were unordered. We treat this as !equal, so
680 a
. or_imm32_reg64(1, m_rScratch
);
684 static ConditionCode
opToConditionCode(Opcode opc
) {
685 using namespace HPHP::Transl
;
688 case JmpGt
: return CC_G
;
689 case JmpGte
: return CC_GE
;
690 case JmpLt
: return CC_L
;
691 case JmpLte
: return CC_LE
;
692 case JmpEq
: return CC_E
;
693 case JmpNeq
: return CC_NE
;
694 case JmpSame
: return CC_E
;
695 case JmpNSame
: return CC_NE
;
696 case JmpInstanceOfBitmask
: return CC_NZ
;
697 case JmpNInstanceOfBitmask
: return CC_Z
;
698 case JmpIsType
: return CC_NZ
;
699 case JmpIsNType
: return CC_Z
;
700 case JmpZero
: return CC_Z
;
701 case JmpNZero
: return CC_NZ
;
702 case ReqBindJmpGt
: return CC_G
;
703 case ReqBindJmpGte
: return CC_GE
;
704 case ReqBindJmpLt
: return CC_L
;
705 case ReqBindJmpLte
: return CC_LE
;
706 case ReqBindJmpEq
: return CC_E
;
707 case ReqBindJmpNeq
: return CC_NE
;
708 case ReqBindJmpSame
: return CC_E
;
709 case ReqBindJmpNSame
: return CC_NE
;
710 case ReqBindJmpInstanceOfBitmask
: return CC_NZ
;
711 case ReqBindJmpNInstanceOfBitmask
: return CC_Z
;
712 case ReqBindJmpZero
: return CC_Z
;
713 case ReqBindJmpNZero
: return CC_NZ
;
719 void CodeGenerator::emitCompare(SSATmp
* src1
, SSATmp
* src2
) {
720 auto const src1Type
= src1
->type();
721 auto const src2Type
= src2
->type();
723 // can't generate CMP instructions correctly for anything that isn't
724 // a bool or a numeric, and we can't mix bool/numerics because
725 // -1 == true in PHP, but not in HHIR binary representation
726 if (!((src1Type
== Type::Int
&& src2Type
== Type::Int
) ||
727 ((src1Type
== Type::Int
|| src1Type
== Type::Dbl
) &&
728 (src2Type
== Type::Int
|| src2Type
== Type::Dbl
)) ||
729 (src1Type
== Type::Bool
&& src2Type
== Type::Bool
) ||
730 (src1Type
== Type::Cls
&& src2Type
== Type::Cls
))) {
731 CG_PUNT(emitCompare
);
733 if (src1Type
== Type::Dbl
|| src2Type
== Type::Dbl
) {
734 PhysReg srcReg1
= prepXMMReg(src1
, m_as
, m_regs
, rCgXMM0
);
735 PhysReg srcReg2
= prepXMMReg(src2
, m_as
, m_regs
, rCgXMM1
);
736 assert(srcReg1
!= rCgXMM1
&& srcReg2
!= rCgXMM0
);
737 doubleCmp(m_as
, srcReg1
, srcReg2
);
739 auto srcReg1
= m_regs
[src1
].reg();
740 auto srcReg2
= m_regs
[src2
].reg();
742 // Note: when both src1 and src2 are constants, we should transform the
743 // branch into an unconditional jump earlier in the IR.
744 if (src1
->isConst()) {
745 // TODO: use compare with immediate or make sure simplifier
746 // canonicalizes this so that constant is src2
747 srcReg1
= m_rScratch
;
748 m_as
. mov_imm64_reg(src1
->getValRawInt(), srcReg1
);
750 if (src2
->isConst()) {
751 if (src1Type
.subtypeOf(Type::Bool
)) {
752 m_as
. cmpb (src2
->getValRawInt(), Reg8(int(srcReg1
)));
754 m_as
. cmp_imm64_reg64(src2
->getValRawInt(), srcReg1
);
757 // Note the reverse syntax in the assembler.
758 // This cmp will compute srcReg1 - srcReg2
759 if (src1Type
.subtypeOf(Type::Bool
)) {
760 m_as
. cmpb (Reg8(int(srcReg2
)), Reg8(int(srcReg1
)));
762 m_as
. cmp_reg64_reg64(srcReg2
, srcReg1
);
768 void CodeGenerator::emitReqBindJcc(ConditionCode cc
,
769 const ReqBindJccData
* extra
) {
771 assert(&m_as
!= &m_astubs
&&
772 "ReqBindJcc only makes sense outside of astubs");
774 prepareForTestAndSmash(a
, 0, TestAndSmashFlags::kAlignJccAndJmp
);
775 auto const patchAddr
= a
.code
.frontier
;
777 auto const jccStub
= m_astubs
.code
.frontier
;
780 // TODO(#2404398): move the setcc into the generic stub code so we
781 // don't need SRFlags::Persistent.
782 a
. setcc (cc
, rbyte(serviceReqArgRegs
[4]));
783 m_tx64
->emitServiceReq(
785 REQ_BIND_JMPCC_FIRST
,
788 uint64_t(extra
->taken
),
789 uint64_t(extra
->notTaken
),
794 a
. jcc (cc
, jccStub
);
798 void CodeGenerator::cgAssertNonNull(IRInstruction
* inst
) {
799 auto srcReg
= m_regs
[inst
->src(0)].reg();
800 auto dstReg
= m_regs
[inst
->dst()].reg();
801 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
803 m_as
.testq (srcReg
, srcReg
);
806 asm_label(m_as
, nonNull
);
808 emitMovRegReg(m_as
, srcReg
, dstReg
);
811 void CodeGenerator::cgAssertType(IRInstruction
* inst
) {
812 auto const& srcRegs
= m_regs
[inst
->src(0)];
813 auto const& dstRegs
= m_regs
[inst
->dst()];
814 shuffle2(m_as
, srcRegs
.reg(0), srcRegs
.reg(1),
815 dstRegs
.reg(0), dstRegs
.reg(1));
818 void CodeGenerator::cgLdUnwinderValue(IRInstruction
* inst
) {
819 cgLoad(rVmTl
, TargetCache::kUnwinderTvOff
, inst
);
822 void CodeGenerator::cgBeginCatch(IRInstruction
* inst
) {
823 auto const& info
= m_state
.catches
[inst
->block()];
824 assert(info
.afterCall
);
826 m_tx64
->registerCatchTrace(info
.afterCall
, m_as
.code
.frontier
);
828 Stats::emitInc(m_as
, Stats::TC_CatchTrace
);
830 // We want to restore state as though the call had completed
831 // successfully, so skip over any stack arguments and pop any
833 if (info
.rspOffset
) {
834 m_as
.addq(info
.rspOffset
, rsp
);
836 PhysRegSaverParity::emitPops(m_as
, info
.savedRegs
);
839 static void unwindResumeHelper(_Unwind_Exception
* data
) {
840 tl_regState
= VMRegState::CLEAN
;
841 _Unwind_Resume(data
);
844 void CodeGenerator::cgEndCatch(IRInstruction
* inst
) {
845 m_as
.cmpb (0, rVmTl
[TargetCache::kUnwinderSideExitOff
]);
846 unlikelyIfBlock(CC_E
,
847 [&](Asm
& as
) { // doSideExit == false, so call _Unwind_Resume
848 as
.loadq(rVmTl
[TargetCache::kUnwinderScratchOff
], rdi
);
849 as
.call ((TCA
)unwindResumeHelper
); // pass control back to the unwinder
853 // doSideExit == true, so fall through to the side exit code
854 Stats::emitInc(m_as
, Stats::TC_CatchSideExit
);
857 void CodeGenerator::cgDeleteUnwinderException(IRInstruction
* inst
) {
858 m_as
.loadq(rVmTl
[TargetCache::kUnwinderScratchOff
], rdi
);
859 m_as
.call ((TCA
)_Unwind_DeleteException
);
862 void CodeGenerator::cgJcc(IRInstruction
* inst
) {
863 emitCompare(inst
->src(0), inst
->src(1));
864 emitFwdJcc(opToConditionCode(inst
->op()), inst
->taken());
867 void CodeGenerator::cgReqBindJcc(IRInstruction
* inst
) {
868 // TODO(#2404427): prepareForTestAndSmash?
869 emitCompare(inst
->src(0), inst
->src(1));
870 emitReqBindJcc(opToConditionCode(inst
->op()),
871 inst
->extra
<ReqBindJccData
>());
875 void CodeGenerator::cgReqBind##x(IRInstruction* i) { cgReqBindJcc(i); } \
876 void CodeGenerator::cg##x (IRInstruction* i) { cgJcc(i); }
891 * Once the arg sources and dests are all assigned; emit moves and exchanges to
892 * put all the args in desired registers. Any arguments that don't fit in
893 * registers will be put on the stack. In addition to moves and exchanges,
894 * shuffleArgs also handles adding lea-offsets for dest registers (dest = src +
895 * lea-offset) and zero extending bools (dest = zeroExtend(src)).
897 typedef Transl::X64Assembler Asm
;
898 static int64_t shuffleArgs(Asm
& a
, ArgGroup
& args
) {
899 // Compute the move/shuffle plan.
901 ArgDesc
* argDescs
[kNumRegs
];
902 memset(moves
, -1, sizeof moves
);
903 memset(argDescs
, 0, sizeof argDescs
);
904 for (size_t i
= 0; i
< args
.numRegArgs(); ++i
) {
905 auto kind
= args
[i
].kind();
906 if (!(kind
== ArgDesc::Kind::Reg
||
907 kind
== ArgDesc::Kind::Addr
||
908 kind
== ArgDesc::Kind::TypeReg
)) {
911 auto dstReg
= args
[i
].dstReg();
912 auto srcReg
= args
[i
].srcReg();
913 if (dstReg
!= srcReg
) {
914 moves
[int(dstReg
)] = int(srcReg
);
915 argDescs
[int(dstReg
)] = &args
[i
];
918 std::vector
<MoveInfo
> howTo
;
919 doRegMoves(moves
, int(rCgGP
), howTo
);
922 for (size_t i
= 0; i
< howTo
.size(); ++i
) {
923 if (howTo
[i
].m_kind
== MoveInfo::Kind::Move
) {
924 if (howTo
[i
].m_reg2
== rCgGP
) {
925 emitMovRegReg(a
, howTo
[i
].m_reg1
, howTo
[i
].m_reg2
);
927 ArgDesc
* argDesc
= argDescs
[int(howTo
[i
].m_reg2
)];
928 ArgDesc::Kind kind
= argDesc
->kind();
929 if (kind
== ArgDesc::Kind::Reg
|| kind
== ArgDesc::Kind::TypeReg
) {
930 if (argDesc
->isZeroExtend()) {
931 assert(howTo
[i
].m_reg1
.isGP());
932 assert(howTo
[i
].m_reg2
.isGP());
933 a
. movzbl (rbyte(howTo
[i
].m_reg1
), r32(howTo
[i
].m_reg2
));
935 emitMovRegReg(a
, howTo
[i
].m_reg1
, howTo
[i
].m_reg2
);
938 assert(kind
== ArgDesc::Kind::Addr
);
939 assert(howTo
[i
].m_reg1
.isGP());
940 assert(howTo
[i
].m_reg2
.isGP());
941 a
. lea (howTo
[i
].m_reg1
[argDesc
->imm().q()],
944 if (kind
!= ArgDesc::Kind::TypeReg
) {
949 assert(howTo
[i
].m_reg1
.isGP());
950 assert(howTo
[i
].m_reg2
.isGP());
951 a
. xchgq (howTo
[i
].m_reg1
, howTo
[i
].m_reg2
);
954 // Handle const-to-register moves, type shifting,
955 // load-effective address and zero extending for bools.
956 // Ignore args that have been handled by the
958 for (size_t i
= 0; i
< args
.numRegArgs(); ++i
) {
959 if (!args
[i
].done()) {
960 ArgDesc::Kind kind
= args
[i
].kind();
961 PhysReg dst
= args
[i
].dstReg();
963 if (kind
== ArgDesc::Kind::Imm
) {
964 a
.emitImmReg(args
[i
].imm().q(), dst
);
965 } else if (kind
== ArgDesc::Kind::TypeReg
) {
966 a
. shlq (kTypeShiftBits
, dst
);
967 } else if (kind
== ArgDesc::Kind::Addr
) {
968 a
. addq (args
[i
].imm(), dst
);
969 } else if (args
[i
].isZeroExtend()) {
970 a
. movzbl (rbyte(dst
), r32(dst
));
971 } else if (RuntimeOption::EvalHHIRGenerateAsserts
&&
972 kind
== ArgDesc::Kind::None
) {
973 a
.emitImmReg(0xbadbadbadbadbad, dst
);
978 // Store any remaining arguments to the stack
979 for (int i
= args
.numStackArgs() - 1; i
>= 0; --i
) {
980 auto& arg
= args
.stk(i
);
981 auto srcReg
= arg
.srcReg();
982 assert(arg
.dstReg() == InvalidReg
);
983 switch (arg
.kind()) {
984 case ArgDesc::Kind::Reg
:
985 if (arg
.isZeroExtend()) {
986 a
. movzbl(rbyte(srcReg
), r32(rCgGP
));
989 if (srcReg
.isXMM()) {
990 emitMovRegReg(a
, srcReg
, rCgGP
);
998 case ArgDesc::Kind::TypeReg
:
999 static_assert(kTypeWordOffset
== 4 || kTypeWordOffset
== 1,
1000 "kTypeWordOffset value not supported");
1001 assert(srcReg
.isGP());
1002 // x86 stacks grow down, so push higher offset items first
1003 if (kTypeWordOffset
== 4) {
1004 a
. pushl(r32(srcReg
));
1005 // 4 bytes of garbage:
1008 // 4 bytes of garbage:
1010 // get the type in the right place in rCgGP before pushing it
1011 a
. movb (rbyte(srcReg
), rbyte(rCgGP
));
1012 a
. shll (CHAR_BIT
, r32(rCgGP
));
1013 a
. pushl(r32(rCgGP
));
1017 case ArgDesc::Kind::Imm
:
1018 a
. emitImmReg(arg
.imm(), rCgGP
);
1022 case ArgDesc::Kind::Addr
:
1025 case ArgDesc::Kind::None
:
1027 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
1028 a
. storeq(0xbadbadbadbadbad, *rsp
);
1033 return args
.numStackArgs() * sizeof(int64_t);
1036 void CodeGenerator::cgCallNative(Asm
& a
, IRInstruction
* inst
) {
1037 using namespace NativeCalls
;
1038 Opcode opc
= inst
->op();
1039 always_assert(CallMap::hasInfo(opc
));
1041 const CallInfo
& info
= CallMap::info(opc
);
1042 ArgGroup
argGroup(m_regs
);
1043 for (auto const& arg
: info
.args
) {
1044 SSATmp
* src
= inst
->src(arg
.srcIdx
);
1050 argGroup
.typedValue(src
);
1053 argGroup
.vectorKeyS(src
);
1056 argGroup
.vectorKeyIS(src
);
1062 switch (info
.func
.type
) {
1064 addr
= info
.func
.ptr
;
1067 addr
= inst
->src(info
.func
.srcIdx
)->getValTCA();
1072 info
.dest
!= DestType::None
? inst
->dst(0) : nullptr,
1078 void CodeGenerator::cgCallHelper(Asm
& a
,
1083 DestType destType
) {
1084 PhysReg dstReg0
= InvalidReg
;
1085 PhysReg dstReg1
= InvalidReg
;
1087 auto &info
= m_regs
[dst
];
1088 dstReg0
= info
.reg(0);
1089 dstReg1
= info
.reg(1);
1091 return cgCallHelper(a
, Transl::CppCall(addr
), dstReg0
, dstReg1
, sync
, args
,
1095 void CodeGenerator::cgCallHelper(Asm
& a
,
1100 DestType destType
) {
1101 cgCallHelper(a
, Transl::CppCall(addr
), dstReg
, InvalidReg
, sync
, args
,
1105 void CodeGenerator::cgCallHelper(Asm
& a
,
1106 const Transl::CppCall
& call
,
1110 DestType destType
) {
1111 cgCallHelper(a
, call
, dstReg
, InvalidReg
, sync
, args
, destType
);
1114 void CodeGenerator::cgCallHelper(Asm
& a
,
1115 const Transl::CppCall
& call
,
1120 DestType destType
) {
1121 cgCallHelper(a
, call
, dstReg0
, dstReg1
, sync
, args
,
1122 m_state
.liveRegs
[m_curInst
], destType
);
1125 void CodeGenerator::cgCallHelper(Asm
& a
,
1126 const Transl::CppCall
& call
,
1132 DestType destType
) {
1133 assert(m_curInst
->isNative());
1135 // Save the caller-saved registers that are live across this
1136 // instruction. The number of regs to save and the number of args
1137 // being passed on the stack affect the parity of the PhysRegSaver,
1138 // so we use the generic version here.
1139 toSave
= toSave
& kCallerSaved
;
1140 assert((toSave
& RegSet().add(dstReg0
).add(dstReg1
)).empty());
1141 PhysRegSaverParity
regSaver(1 + args
.numStackArgs(), a
, toSave
);
1143 // Assign registers to the arguments then prepare them for the call.
1144 for (size_t i
= 0; i
< args
.numRegArgs(); i
++) {
1145 args
[i
].setDstReg(argNumToRegName
[i
]);
1147 regSaver
.bytesPushed(shuffleArgs(a
, args
));
1149 // do the call; may use a trampoline
1150 m_tx64
->emitCall(a
, call
);
1151 if (sync
!= SyncOptions::kNoSyncPoint
) {
1152 recordSyncPoint(a
, sync
);
1155 if (m_state
.catchTrace
) {
1156 auto& info
= m_state
.catches
[m_state
.catchTrace
->front()];
1157 assert(!info
.afterCall
);
1158 info
.afterCall
= a
.code
.frontier
;
1159 info
.savedRegs
= toSave
;
1160 info
.rspOffset
= regSaver
.rspAdjustment();
1163 // copy the call result to the destination register(s)
1164 if (destType
== DestType::TV
) {
1165 // rax contains m_type and m_aux but we're expecting just the
1166 // type in the lower bits, so shift the type result register.
1167 auto rval
= packed_tv
? reg::rdx
: reg::rax
;
1168 auto rtyp
= packed_tv
? reg::rax
: reg::rdx
;
1169 if (kTypeShiftBits
> 0) a
.shrq(kTypeShiftBits
, rtyp
);
1170 shuffle2(a
, rval
, rtyp
, dstReg0
, dstReg1
);
1171 } else if (destType
== DestType::SSA
) {
1172 // copy the single-register result to dstReg0
1173 assert(dstReg1
== InvalidReg
);
1174 if (dstReg0
!= InvalidReg
) emitMovRegReg(a
, reg::rax
, dstReg0
);
1176 // void return type, no registers have values
1177 assert(dstReg0
== InvalidReg
&& dstReg1
== InvalidReg
);
1181 void CodeGenerator::cgMov(IRInstruction
* inst
) {
1182 assert(!m_regs
[inst
->src(0)].hasReg(1));//TODO: t2082361: handle Gen & Cell
1183 SSATmp
* dst
= inst
->dst();
1184 SSATmp
* src
= inst
->src(0);
1185 auto dstReg
= m_regs
[dst
].reg();
1186 if (!m_regs
[src
].hasReg(0)) {
1187 assert(src
->isConst());
1188 if (src
->type() == Type::Bool
) {
1189 emitLoadImm(m_as
, (int64_t)src
->getValBool(), dstReg
);
1191 emitLoadImm(m_as
, src
->getValRawInt(), dstReg
);
1194 auto srcReg
= m_regs
[src
].reg();
1195 emitMovRegReg(m_as
, srcReg
, dstReg
);
1199 template<class OpInstr
, class Oper
>
1200 void CodeGenerator::cgUnaryIntOp(SSATmp
* dst
,
1204 if (src
->type() != Type::Int
&& src
->type() != Type::Bool
) {
1205 assert(0); CG_PUNT(UnaryIntOp
);
1207 auto dstReg
= m_regs
[dst
].reg();
1208 auto srcReg
= m_regs
[src
].reg();
1209 assert(dstReg
!= InvalidReg
);
1212 // Integer operations require 64-bit representations
1213 zeroExtendIfBool(a
, src
, m_regs
[src
]);
1215 if (srcReg
!= InvalidReg
) {
1216 emitMovRegReg(a
, srcReg
, dstReg
);
1217 (a
.*instr
) (dstReg
);
1219 assert(src
->isConst());
1220 emitLoadImm(a
, oper(src
->getValRawInt()), dstReg
);
1224 void CodeGenerator::cgNegateWork(SSATmp
* dst
, SSATmp
* src
) {
1225 cgUnaryIntOp(dst
, src
, &Asm::neg
, [](int64_t i
) { return -i
; });
1228 inline static Reg8
convertToReg8(PhysReg reg
) { return rbyte(reg
); }
1229 inline static Reg64
convertToReg64(PhysReg reg
) { return reg
; }
1231 template<class Oper
, class RegType
>
1232 void CodeGenerator::cgBinaryIntOp(IRInstruction
* inst
,
1233 void (Asm::*instrIR
)(Immed
, RegType
),
1234 void (Asm::*instrRR
)(RegType
, RegType
),
1235 void (Asm::*movInstr
)(RegType
, RegType
),
1237 RegType (*convertReg
)(PhysReg
),
1238 Commutativity commuteFlag
) {
1239 const SSATmp
* dst
= inst
->dst();
1240 const SSATmp
* src1
= inst
->src(0);
1241 const SSATmp
* src2
= inst
->src(1);
1242 if (!(src1
->isA(Type::Bool
) || src1
->isA(Type::Int
)) ||
1243 !(src2
->isA(Type::Bool
) || src2
->isA(Type::Int
))) {
1244 CG_PUNT(cgBinaryIntOp
);
1247 bool const commutative
= commuteFlag
== Commutative
;
1248 auto const dstReg
= m_regs
[dst
].reg();
1249 auto const src1Reg
= m_regs
[src1
].reg();
1250 auto const src2Reg
= m_regs
[src2
].reg();
1253 auto const dstOpReg
= convertReg(dstReg
);
1254 auto const src1OpReg
= convertReg(src1Reg
);
1255 auto const src2OpReg
= convertReg(src2Reg
);
1256 auto const rOpScratch
= convertReg(m_rScratch
);
1259 if (src1Reg
!= InvalidReg
&& src2Reg
!= InvalidReg
) {
1260 if (dstReg
== src1Reg
) {
1261 (a
.*instrRR
) (src2OpReg
, dstOpReg
);
1262 } else if (dstReg
== src2Reg
) {
1264 (a
.*instrRR
) (src1OpReg
, dstOpReg
);
1266 (a
.*movInstr
)(src1OpReg
, rOpScratch
);
1267 (a
.*instrRR
) (src2OpReg
, rOpScratch
);
1268 (a
.*movInstr
)(rOpScratch
, dstOpReg
);
1271 emitMovRegReg(a
, src1Reg
, dstReg
);
1272 (a
.*instrRR
) (src2OpReg
, dstOpReg
);
1278 if (src1Reg
== InvalidReg
&& src2Reg
== InvalidReg
) {
1279 assert(src1
->isConst() && src2
->isConst());
1280 int64_t value
= oper(src1
->getValRawInt(), src2
->getValRawInt());
1281 emitLoadImm(a
, value
, dstReg
);
1285 // One register, and one immediate.
1287 auto immedSrc
= (src2Reg
== InvalidReg
? src2
: src1
);
1288 auto immed
= immedSrc
->getValRawInt();
1289 auto srcReg
= m_regs
[(src2Reg
== InvalidReg
? src1
: src2
)].reg();
1290 if (srcReg
== dstReg
) {
1291 (a
.*instrIR
) (immed
, dstOpReg
);
1293 emitLoadImm(a
, immed
, dstReg
);
1294 (a
.*instrRR
) (convertReg(srcReg
), dstOpReg
);
1300 if (src1Reg
== InvalidReg
) {
1301 if (dstReg
== src2Reg
) {
1302 emitLoadImm(a
, src1
->getValRawInt(), m_rScratch
);
1303 (a
.*instrRR
) (src2OpReg
, rOpScratch
);
1304 (a
.*movInstr
)(rOpScratch
, dstOpReg
);
1306 emitLoadImm(a
, src1
->getValRawInt(), dstReg
);
1307 (a
.*instrRR
) (src2OpReg
, dstOpReg
);
1312 assert(src2Reg
== InvalidReg
);
1313 emitMovRegReg(a
, src1Reg
, dstReg
);
1314 (a
.*instrIR
) (src2
->getValRawInt(), dstOpReg
);
1317 template<class Oper
, class RegType
>
1318 void CodeGenerator::cgBinaryOp(IRInstruction
* inst
,
1319 void (Asm::*instrIR
)(Immed
, RegType
),
1320 void (Asm::*instrRR
)(RegType
, RegType
),
1321 void (Asm::*movInstr
)(RegType
, RegType
),
1322 void (Asm::*fpInstr
)(RegXMM
, RegXMM
),
1324 RegType (*convertReg
)(PhysReg
),
1325 Commutativity commuteFlag
) {
1326 const SSATmp
* dst
= inst
->dst();
1327 const SSATmp
* src1
= inst
->src(0);
1328 const SSATmp
* src2
= inst
->src(1);
1329 if (!(src1
->isA(Type::Bool
) || src1
->isA(Type::Int
) || src1
->isA(Type::Dbl
))
1331 !(src2
->isA(Type::Bool
) || src2
->isA(Type::Int
) || src2
->isA(Type::Dbl
)) )
1333 CG_PUNT(cgBinaryOp
);
1335 if (src1
->isA(Type::Dbl
) || src2
->isA(Type::Dbl
)) {
1336 PhysReg dstReg
= m_regs
[dst
].reg();
1337 PhysReg resReg
= dstReg
.isXMM() && dstReg
!= m_regs
[src2
].reg() ?
1338 dstReg
: PhysReg(rCgXMM0
);
1339 assert(resReg
.isXMM());
1341 PhysReg srcReg1
= prepXMMReg(src1
, m_as
, m_regs
, resReg
);
1342 PhysReg srcReg2
= prepXMMReg(src2
, m_as
, m_regs
, rCgXMM1
);
1343 assert(srcReg1
!= rCgXMM1
&& srcReg2
!= rCgXMM0
);
1345 emitMovRegReg(m_as
, srcReg1
, resReg
);
1347 (m_as
.*fpInstr
)(srcReg2
, resReg
);
1349 emitMovRegReg(m_as
, resReg
, dstReg
);
1352 cgBinaryIntOp(inst
, instrIR
, instrRR
, movInstr
,
1353 oper
, convertReg
, commuteFlag
);
1356 bool CodeGenerator::emitIncDecHelper(SSATmp
* dst
, SSATmp
* src1
, SSATmp
* src2
,
1357 void(Asm::*emitFunc
)(Reg64
)) {
1358 if (m_regs
[src1
].reg() != InvalidReg
&&
1359 m_regs
[dst
].reg() != InvalidReg
&&
1360 src1
->isA(Type::Int
) &&
1362 src2
->isConst() && src2
->isA(Type::Int
) && src2
->getValInt() == 1) {
1363 emitMovRegReg(m_as
, m_regs
[src1
].reg(), m_regs
[dst
].reg());
1364 (m_as
.*emitFunc
)(m_regs
[dst
].reg());
1371 * If src2 is 1, this generates dst = src1 + 1 using the "inc" x86 instruction.
1372 * The return value is whether or not the instruction could be generated.
1374 bool CodeGenerator::emitInc(SSATmp
* dst
, SSATmp
* src1
, SSATmp
* src2
) {
1375 return emitIncDecHelper(dst
, src1
, src2
, &Asm::incq
);
1379 * If src2 is 1, this generates dst = src1 - 1 using the "dec" x86 instruction.
1380 * The return value is whether or not the instruction could be generated.
1382 bool CodeGenerator::emitDec(SSATmp
* dst
, SSATmp
* src1
, SSATmp
* src2
) {
1383 return emitIncDecHelper(dst
, src1
, src2
, &Asm::decq
);
1386 void CodeGenerator::cgOpAdd(IRInstruction
* inst
) {
1387 SSATmp
* dst
= inst
->dst();
1388 SSATmp
* src1
= inst
->src(0);
1389 SSATmp
* src2
= inst
->src(1);
1391 // Special cases: x = y + 1
1392 if (emitInc(dst
, src1
, src2
) || emitInc(dst
, src2
, src1
)) return;
1398 &Asm::addsd_xmm_xmm
,
1399 std::plus
<int64_t>(),
1404 void CodeGenerator::cgOpSub(IRInstruction
* inst
) {
1405 SSATmp
* dst
= inst
->dst();
1406 SSATmp
* src1
= inst
->src(0);
1407 SSATmp
* src2
= inst
->src(1);
1409 if (emitDec(dst
, src1
, src2
)) return;
1411 if (src1
->isConst() && src1
->isA(Type::Int
) && src1
->getValInt() == 0 &&
1412 src2
->isA(Type::Int
)) {
1413 cgNegateWork(dst
, src2
);
1421 &Asm::subsd_xmm_xmm
,
1422 std::minus
<int64_t>(),
1427 void CodeGenerator::cgOpDiv(IRInstruction
* inst
) {
1431 void CodeGenerator::cgOpBitAnd(IRInstruction
* inst
) {
1436 [] (int64_t a
, int64_t b
) { return a
& b
; },
1441 void CodeGenerator::cgOpBitOr(IRInstruction
* inst
) {
1446 [] (int64_t a
, int64_t b
) { return a
| b
; },
1451 void CodeGenerator::cgOpBitXor(IRInstruction
* inst
) {
1456 [] (int64_t a
, int64_t b
) { return a
^ b
; },
1461 void CodeGenerator::cgOpBitNot(IRInstruction
* inst
) {
1462 cgUnaryIntOp(inst
->dst(),
1465 [](int64_t i
) { return ~i
; });
1468 void CodeGenerator::cgOpLogicXor(IRInstruction
* inst
) {
1473 [] (bool a
, bool b
) { return a
^ b
; },
1478 void CodeGenerator::cgOpMul(IRInstruction
* inst
) {
1483 &Asm::mulsd_xmm_xmm
,
1484 std::multiplies
<int64_t>(),
1489 void CodeGenerator::cgOpNot(IRInstruction
* inst
) {
1490 auto const src
= inst
->src(0);
1491 auto const dstReg
= m_regs
[inst
->dst()].reg();
1494 if (src
->isConst()) {
1495 a
. movb (!src
->getValBool(), rbyte(dstReg
));
1497 if (dstReg
!= m_regs
[src
].reg()) {
1498 a
. movb (rbyte(m_regs
[src
].reg()), rbyte(dstReg
));
1500 a
. xorb (1, rbyte(dstReg
));
1504 ///////////////////////////////////////////////////////////////////////////////
1505 // Comparison Operators
1506 ///////////////////////////////////////////////////////////////////////////////
1508 #define DISPATCHER(name) \
1509 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, StringData* a2) \
1510 { return name(a1, a2); } \
1511 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, int64_t a2) \
1512 { return name(a1, a2); } \
1513 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, ObjectData* a2) \
1514 { return name(a1, Object(a2)); } \
1515 HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, ObjectData* a2) \
1516 { return name(Object(a1), Object(a2)); } \
1517 HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, int64_t a2) \
1518 { return name(Object(a1), a2); } \
1519 HOT_FUNC_VM int64_t ccmp_ ## name (ArrayData* a1, ArrayData* a2) \
1520 { return name(Array(a1), Array(a2)); }
1529 template <typename A
, typename B
>
1530 inline int64_t ccmp_nsame(A a
, B b
) { return !ccmp_same(a
, b
); }
1532 template <typename A
, typename B
>
1533 inline int64_t ccmp_nequal(A a
, B b
) { return !ccmp_equal(a
, b
); }
1535 template <typename A
, typename B
>
1536 inline int64_t ccmp_lte(A a
, B b
) { return !ccmp_more(a
, b
); }
1538 template <typename A
, typename B
>
1539 inline int64_t ccmp_gte(A a
, B b
) { return !ccmp_less(a
, b
); }
1541 #define CG_OP_CMP(inst, setter, name) \
1542 cgOpCmpHelper(inst, &Asm:: setter, ccmp_ ## name, ccmp_ ## name, \
1543 ccmp_ ## name, ccmp_ ## name, ccmp_ ## name, ccmp_ ## name)
1545 // SRON - string, resource, object, or number
1546 static bool typeIsSRON(Type t
) {
1548 || t
== Type::Obj
// encompases object and resource
1554 void CodeGenerator::cgOpCmpHelper(
1555 IRInstruction
* inst
,
1556 void (Asm::*setter
)(Reg8
),
1557 int64_t (*str_cmp_str
)(StringData
*, StringData
*),
1558 int64_t (*str_cmp_int
)(StringData
*, int64_t),
1559 int64_t (*str_cmp_obj
)(StringData
*, ObjectData
*),
1560 int64_t (*obj_cmp_obj
)(ObjectData
*, ObjectData
*),
1561 int64_t (*obj_cmp_int
)(ObjectData
*, int64_t),
1562 int64_t (*arr_cmp_arr
)( ArrayData
*, ArrayData
*)
1564 SSATmp
* dst
= inst
->dst();
1565 SSATmp
* src1
= inst
->src(0);
1566 SSATmp
* src2
= inst
->src(1);
1568 Type type1
= src1
->type();
1569 Type type2
= src2
->type();
1571 auto src1Reg
= m_regs
[src1
].reg();
1572 auto src2Reg
= m_regs
[src2
].reg();
1573 auto dstReg
= m_regs
[dst
].reg();
1575 auto setFromFlags
= [&] {
1576 (m_as
.*setter
)(rbyte(dstReg
));
1578 // It is possible that some pass has been done after simplification; if such
1579 // a pass invalidates our invariants, then just punt.
1581 // simplifyCmp has done const-const optimization
1583 // If the types are the same and there is only one constant,
1584 // simplifyCmp has moved it to the right.
1585 if (src1
->isConst()) {
1586 CG_PUNT(cgOpCmpHelper_const
);
1589 /////////////////////////////////////////////////////////////////////////////
1590 // case 1: null/string cmp string
1591 // simplifyCmp has converted the null to ""
1592 if (type1
.isString() && type2
.isString()) {
1593 ArgGroup
args(m_regs
);
1594 args
.ssa(src1
).ssa(src2
);
1595 cgCallHelper(m_as
, (TCA
)str_cmp_str
, dst
, SyncOptions::kSyncPoint
, args
);
1598 /////////////////////////////////////////////////////////////////////////////
1599 // case 2: bool/null cmp anything
1600 // simplifyCmp has converted all args to bool
1601 else if (type1
== Type::Bool
&& type2
== Type::Bool
) {
1602 if (src2
->isConst()) {
1603 m_as
. cmpb (src2
->getValBool(), Reg8(int(src1Reg
)));
1605 m_as
. cmpb (Reg8(int(src2Reg
)), Reg8(int(src1Reg
)));
1610 /////////////////////////////////////////////////////////////////////////////
1611 // case 3, 4, and 7: string/resource/object/number (sron) cmp sron
1612 // These cases must be amalgamated because Type::Obj can refer to an object
1613 // or to a resource.
1614 // strings are canonicalized to the left, ints to the right
1615 else if (typeIsSRON(type1
) && typeIsSRON(type2
)) {
1616 // the common case: int cmp int
1617 if (type1
== Type::Int
&& type2
== Type::Int
) {
1618 if (src2
->isConst()) {
1619 m_as
.cmp_imm64_reg64(src2
->getValInt(), src1Reg
);
1621 m_as
.cmp_reg64_reg64(src2Reg
, src1Reg
);
1626 else if (type1
== Type::Dbl
|| type2
== Type::Dbl
) {
1627 if ((type1
== Type::Dbl
|| type1
== Type::Int
) &&
1628 (type2
== Type::Dbl
|| type2
== Type::Int
)) {
1629 PhysReg srcReg1
= prepXMMReg(src1
, m_as
, m_regs
, rCgXMM0
);
1630 PhysReg srcReg2
= prepXMMReg(src2
, m_as
, m_regs
, rCgXMM1
);
1631 assert(srcReg1
!= rCgXMM1
&& srcReg2
!= rCgXMM0
);
1632 doubleCmp(m_as
, srcReg1
, srcReg2
);
1635 CG_PUNT(cgOpCmpHelper_Dbl
);
1639 else if (type1
.isString()) {
1640 // string cmp string is dealt with in case 1
1641 // string cmp double is punted above
1643 if (type2
== Type::Int
) {
1644 ArgGroup
args(m_regs
);
1645 args
.ssa(src1
).ssa(src2
);
1646 cgCallHelper(m_as
, (TCA
)str_cmp_int
, dst
,
1647 SyncOptions::kSyncPoint
, args
);
1648 } else if (type2
== Type::Obj
) {
1649 ArgGroup
args(m_regs
);
1650 args
.ssa(src1
).ssa(src2
);
1651 cgCallHelper(m_as
, (TCA
)str_cmp_obj
, dst
,
1652 SyncOptions::kSyncPoint
, args
);
1654 CG_PUNT(cgOpCmpHelper_sx
);
1658 else if (type1
== Type::Obj
) {
1659 // string cmp object/resource is dealt with above
1660 // object cmp double is punted above
1662 if (type2
== Type::Obj
) {
1663 ArgGroup
args(m_regs
);
1664 args
.ssa(src1
).ssa(src2
);
1665 cgCallHelper(m_as
, (TCA
)obj_cmp_obj
, dst
,
1666 SyncOptions::kSyncPoint
, args
);
1667 } else if (type2
== Type::Int
) {
1668 ArgGroup
args(m_regs
);
1669 args
.ssa(src1
).ssa(src2
);
1670 cgCallHelper(m_as
, (TCA
)obj_cmp_int
, dst
,
1671 SyncOptions::kSyncPoint
, args
);
1673 CG_PUNT(cgOpCmpHelper_ox
);
1680 /////////////////////////////////////////////////////////////////////////////
1681 // case 5: array cmp array
1682 else if (type1
.isArray() && type2
.isArray()) {
1683 ArgGroup
args(m_regs
);
1684 args
.ssa(src1
).ssa(src2
);
1685 cgCallHelper(m_as
, (TCA
)arr_cmp_arr
, dst
, SyncOptions::kSyncPoint
, args
);
1688 /////////////////////////////////////////////////////////////////////////////
1689 // case 6: array cmp anything
1690 // simplifyCmp has already dealt with this case.
1692 /////////////////////////////////////////////////////////////////////////////
1694 // We have a type which is not a common type. It might be a cell or a box.
1695 CG_PUNT(cgOpCmpHelper_unimplemented
);
1699 void CodeGenerator::cgOpEq(IRInstruction
* inst
) {
1700 CG_OP_CMP(inst
, sete
, equal
);
1703 void CodeGenerator::cgOpNeq(IRInstruction
* inst
) {
1704 CG_OP_CMP(inst
, setne
, nequal
);
1707 void CodeGenerator::cgOpSame(IRInstruction
* inst
) {
1708 CG_OP_CMP(inst
, sete
, same
);
1711 void CodeGenerator::cgOpNSame(IRInstruction
* inst
) {
1712 CG_OP_CMP(inst
, setne
, nsame
);
1715 void CodeGenerator::cgOpLt(IRInstruction
* inst
) {
1716 CG_OP_CMP(inst
, setl
, less
);
1719 void CodeGenerator::cgOpGt(IRInstruction
* inst
) {
1720 CG_OP_CMP(inst
, setg
, more
);
1723 void CodeGenerator::cgOpLte(IRInstruction
* inst
) {
1724 CG_OP_CMP(inst
, setle
, lte
);
1727 void CodeGenerator::cgOpGte(IRInstruction
* inst
) {
1728 CG_OP_CMP(inst
, setge
, gte
);
1731 ///////////////////////////////////////////////////////////////////////////////
1732 // Type check operators
1733 ///////////////////////////////////////////////////////////////////////////////
1735 // Overloads to put the ObjectData* into a register so emitTypeTest
1736 // can cmp to the Class* expected by the specialized Type
1738 // Nothing to do, return the register that contain the ObjectData already
1739 Reg64
getObjectDataEnregistered(Asm
& as
, PhysReg dataSrc
, Reg64 scratch
) {
1743 // Enregister the meoryRef so it can be used with an offset by the
1745 Reg64
getObjectDataEnregistered(Asm
& as
,
1748 as
.loadq(dataSrc
, scratch
);
1752 template<class Loc1
, class Loc2
, class JmpFn
>
1753 void CodeGenerator::emitTypeTest(Type type
, Loc1 typeSrc
, Loc2 dataSrc
,
1755 assert(!type
.subtypeOf(Type::Cls
));
1757 if (type
.isString()) {
1758 emitTestTVType(m_as
, KindOfStringBit
, typeSrc
);
1760 } else if (type
.equals(Type::UncountedInit
)) {
1761 emitTestTVType(m_as
, KindOfUncountedInitBit
, typeSrc
);
1763 } else if (type
.equals(Type::Uncounted
)) {
1764 emitCmpTVType(m_as
, KindOfRefCountThreshold
, typeSrc
);
1766 } else if (type
.equals(Type::Cell
)) {
1767 emitCmpTVType(m_as
, KindOfRef
, typeSrc
);
1769 } else if (type
.equals(Type::Gen
)) {
1773 assert(type
.isKnownDataType());
1774 DataType dataType
= type
.toDataType();
1775 assert(dataType
== KindOfRef
||
1776 (dataType
>= KindOfUninit
&& dataType
<= KindOfObject
));
1777 emitCmpTVType(m_as
, dataType
, typeSrc
);
1781 if (type
.strictSubtypeOf(Type::Obj
)) {
1782 // emit the specific class test
1783 assert(type
.getClass()->attrs() & AttrFinal
);
1784 auto reg
= getObjectDataEnregistered(m_as
, dataSrc
, m_rScratch
);
1785 m_as
.cmpq(type
.getClass(), reg
[ObjectData::getVMClassOffset()]);
1790 template<class JmpFn
>
1791 void CodeGenerator::emitIsTypeTest(IRInstruction
* inst
, JmpFn doJcc
) {
1792 auto const src
= inst
->src(0);
1794 // punt if specialized object for now
1795 if (inst
->typeParam().strictSubtypeOf(Type::Obj
)) {
1796 CG_PUNT(IsType
-SpecializedUnsupported
);
1798 if (inst
->typeParam().equals(Type::Obj
)) {
1799 auto const srcReg
= m_regs
[src
].reg();
1800 if (src
->isA(Type::PtrToGen
)) {
1801 emitTestTVType(m_as
, KindOfObject
, srcReg
[TVOFF(m_type
)]);
1802 TCA toPatch
= m_as
.code
.frontier
;
1803 m_as
. jne8(toPatch
); // 1
1805 // Get the ObjectData*
1806 emitDeref(m_as
, srcReg
, m_rScratch
);
1807 m_as
. cmpq(SystemLib::s_resourceClass
,
1808 m_rScratch
[ObjectData::getVMClassOffset()]);
1810 m_as
.patchJcc8(toPatch
, m_as
.code
.frontier
);
1812 // Cases where src isn't an Obj should have been simplified away
1813 if (!src
->isA(Type::Obj
)) {
1814 CG_PUNT(IsType
-KnownWrongType
);
1816 m_as
. cmpq(SystemLib::s_resourceClass
,
1817 srcReg
[ObjectData::getVMClassOffset()]);
1819 // At this point, the flags say "equal" if is_object is false.
1824 if (src
->isA(Type::PtrToGen
)) {
1825 PhysReg base
= m_regs
[src
].reg();
1826 emitTypeTest(inst
->typeParam(), base
[TVOFF(m_type
)],
1827 base
[TVOFF(m_data
)],
1828 [&](ConditionCode cc
) { doJcc(cc
); });
1831 assert(src
->isA(Type::Gen
));
1832 assert(!src
->isConst());
1834 PhysReg typeSrcReg
= m_regs
[src
].reg(1); // type register
1835 if (typeSrcReg
== InvalidReg
) {
1836 CG_PUNT(IsType
-KnownType
);
1838 PhysReg dataSrcReg
= m_regs
[src
].reg(); // data register
1839 emitTypeTest(inst
->typeParam(), typeSrcReg
, dataSrcReg
,
1840 [&](ConditionCode cc
) { doJcc(cc
); });
1844 void CodeGenerator::emitTypeCheck(Type type
,
1848 emitTypeTest(type
, typeSrc
, dataSrc
,
1849 [&](ConditionCode cc
) {
1850 emitFwdJcc(ccNegate(cc
), taken
);
1855 void CodeGenerator::emitTypeGuard(Type type
, Loc typeSrc
, Loc dataSrc
) {
1856 emitTypeTest(type
, typeSrc
, dataSrc
,
1857 [&](ConditionCode cc
) {
1858 auto const destSK
= SrcKey(curFunc(), m_curTrace
->bcOff());
1859 auto const destSR
= m_tx64
->getSrcRec(destSK
);
1860 m_tx64
->emitFallbackCondJmp(m_as
, *destSR
, ccNegate(cc
));
1864 void CodeGenerator::emitSetCc(IRInstruction
* inst
, ConditionCode cc
) {
1865 m_as
.setcc(cc
, rbyte(m_regs
[inst
->dst()].reg()));
1868 void CodeGenerator::cgIsTypeMemCommon(IRInstruction
* inst
, bool negate
) {
1869 bool called
= false; // check emitSetCc is called only once
1870 emitIsTypeTest(inst
,
1871 [&](ConditionCode cc
) {
1873 emitSetCc(inst
, negate
? ccNegate(cc
) : cc
);
1878 void CodeGenerator::cgIsTypeCommon(IRInstruction
* inst
, bool negate
) {
1879 bool called
= false; // check emitSetCc is called only once
1880 emitIsTypeTest(inst
,
1881 [&](ConditionCode cc
) {
1883 emitSetCc(inst
, negate
? ccNegate(cc
) : cc
);
1888 void CodeGenerator::cgJmpIsTypeCommon(IRInstruction
* inst
, bool negate
) {
1889 emitIsTypeTest(inst
,
1890 [&](ConditionCode cc
) {
1891 emitFwdJcc(negate
? ccNegate(cc
) : cc
, inst
->taken());
1895 void CodeGenerator::cgIsType(IRInstruction
* inst
) {
1896 cgIsTypeCommon(inst
, false);
1899 void CodeGenerator::cgIsNType(IRInstruction
* inst
) {
1900 cgIsTypeCommon(inst
, true);
1903 // TODO(#2404341): remove JmpIs{N,}Type
1905 void CodeGenerator::cgJmpIsType(IRInstruction
* inst
) {
1906 cgJmpIsTypeCommon(inst
, false);
1909 void CodeGenerator::cgJmpIsNType(IRInstruction
* inst
) {
1910 cgJmpIsTypeCommon(inst
, true);
1913 void CodeGenerator::cgIsTypeMem(IRInstruction
* inst
) {
1914 cgIsTypeMemCommon(inst
, false);
1917 void CodeGenerator::cgIsNTypeMem(IRInstruction
* inst
) {
1918 cgIsTypeMemCommon(inst
, true);
1921 ///////////////////////////////////////////////////////////////////////////////
1923 HOT_FUNC_VM
static bool instanceOfHelper(const Class
* objClass
,
1924 const Class
* testClass
) {
1925 return testClass
&& objClass
->classof(testClass
);
1928 void CodeGenerator::cgInstanceOf(IRInstruction
* inst
) {
1930 TCA(instanceOfHelper
),
1932 SyncOptions::kNoSyncPoint
,
1935 .ssa(inst
->src(1)));
1939 * Check instanceof using instance bitmasks.
1941 * Note it's not necessary to check whether the test class is defined:
1942 * if it doesn't exist than the candidate can't be an instance of it
1943 * and will fail this check.
1945 void CodeGenerator::emitInstanceBitmaskCheck(IRInstruction
* inst
) {
1946 auto const rObjClass
= m_regs
[inst
->src(0)].reg(0);
1947 auto const testClassName
= inst
->src(1)->getValStr();
1952 if (!Class::getInstanceBitMask(testClassName
, offset
, mask
)) {
1953 always_assert(!"cgInstanceOfBitmask had no bitmask");
1955 a
. testb (int8_t(mask
), rObjClass
[offset
]);
1958 void CodeGenerator::cgInstanceOfBitmask(IRInstruction
* inst
) {
1960 emitInstanceBitmaskCheck(inst
);
1961 a
. setnz (rbyte(m_regs
[inst
->dst()].reg()));
1964 void CodeGenerator::cgNInstanceOfBitmask(IRInstruction
* inst
) {
1966 emitInstanceBitmaskCheck(inst
);
1967 a
. setz (rbyte(m_regs
[inst
->dst()].reg()));
1970 void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction
* inst
) {
1971 emitInstanceBitmaskCheck(inst
);
1972 emitFwdJcc(CC_NZ
, inst
->taken());
1975 void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction
* inst
) {
1976 emitInstanceBitmaskCheck(inst
);
1977 emitFwdJcc(CC_Z
, inst
->taken());
1980 void CodeGenerator::cgReqBindJmpInstanceOfBitmask(IRInstruction
* inst
) {
1981 emitInstanceBitmaskCheck(inst
);
1982 emitReqBindJcc(opToConditionCode(inst
->op()),
1983 inst
->extra
<ReqBindJccData
>());
1986 void CodeGenerator::cgReqBindJmpNInstanceOfBitmask(IRInstruction
* inst
) {
1987 emitInstanceBitmaskCheck(inst
);
1988 emitReqBindJcc(opToConditionCode(inst
->op()),
1989 inst
->extra
<ReqBindJccData
>());
1993 * Check instanceof using the superclass vector on the end of the
1996 void CodeGenerator::cgExtendsClass(IRInstruction
* inst
) {
1997 auto const rObjClass
= m_regs
[inst
->src(0)].reg();
1998 auto const testClass
= inst
->src(1)->getValClass();
1999 auto rTestClass
= m_regs
[inst
->src(1)].reg();
2000 auto const rdst
= rbyte(m_regs
[inst
->dst()].reg());
2007 if (rTestClass
== InvalidReg
) { // TODO(#2031606)
2008 rTestClass
= m_rScratch
; // careful below about asm-x64 smashing this
2009 emitLoadImm(a
, (int64_t)testClass
, rTestClass
);
2012 // Test if it is the exact same class. TODO(#2044801): we should be
2013 // doing this control flow at the IR level.
2014 if (!(testClass
->attrs() & AttrAbstract
)) {
2015 a
. cmpq (rTestClass
, rObjClass
);
2021 auto const vecOffset
= Class::classVecOff() +
2022 sizeof(Class
*) * (testClass
->classVecLen() - 1);
2024 // Check the length of the class vectors---if the candidate's is at
2025 // least as long as the potential base (testClass) it might be a
2027 asm_label(a
, notExact
);
2028 a
. cmpl (testClass
->classVecLen(),
2029 rObjClass
[Class::classVecLenOff()]);
2030 a
. jb8 (falseLabel
);
2032 // If it's a subclass, rTestClass must be at the appropriate index.
2033 a
. cmpq (rTestClass
, rObjClass
[vecOffset
]);
2037 asm_label(a
, falseLabel
);
2038 a
. xorl (r32(rdst
), r32(rdst
));
2043 void CodeGenerator::cgConvDblToBool(IRInstruction
* inst
) {
2044 SSATmp
* dst
= inst
->dst();
2045 auto dstReg
= m_regs
[dst
].reg();
2046 assert(dstReg
!= InvalidReg
);
2047 SSATmp
* src
= inst
->src(0);
2048 auto srcReg
= m_regs
[src
].reg();
2049 if (srcReg
== InvalidReg
) {
2050 assert(src
->isConst());
2051 double constVal
= src
->getValDbl();
2052 if (constVal
== 0.0) {
2053 m_as
.xor_reg64_reg64(dstReg
, dstReg
);
2055 m_as
.mov_imm64_reg(1, dstReg
);
2058 emitMovRegReg(m_as
, srcReg
, dstReg
);
2059 m_as
.shlq(1, dstReg
); // 0.0 stays zero and -0.0 is now 0.0
2060 m_as
.setne(rbyte(dstReg
)); // lower byte becomes 1 if dstReg != 0
2061 m_as
.movzbl(rbyte(dstReg
), r32(dstReg
));
2065 void CodeGenerator::cgConvIntToBool(IRInstruction
* inst
) {
2066 SSATmp
* dst
= inst
->dst();
2067 auto dstReg
= m_regs
[dst
].reg();
2068 assert(dstReg
!= InvalidReg
);
2069 SSATmp
* src
= inst
->src(0);
2070 auto srcReg
= m_regs
[src
].reg();
2072 if (srcReg
== InvalidReg
) {
2073 assert(src
->isConst());
2074 int64_t constVal
= src
->getValInt();
2075 if (constVal
== 0) {
2076 m_as
.xor_reg64_reg64(dstReg
, dstReg
);
2078 m_as
.mov_imm64_reg(1, dstReg
);
2081 m_as
.test_reg64_reg64(srcReg
, srcReg
);
2082 m_as
.setne(rbyte(dstReg
));
2083 m_as
.movzbl(rbyte(dstReg
), r32(dstReg
));
2087 void CodeGenerator::emitConvBoolOrIntToDbl(IRInstruction
* inst
) {
2088 SSATmp
* src
= inst
->src(0);
2089 SSATmp
* dst
= inst
->dst();
2090 PhysReg dstReg
= m_regs
[dst
].reg();
2091 assert(src
->isA(Type::Bool
) || src
->isA(Type::Int
));
2092 assert(dstReg
!= InvalidReg
);
2093 if (src
->isConst()) {
2094 int64_t constVal
= src
->getValRawInt();
2095 if (src
->isA(Type::Bool
)) constVal
= constVal
!= 0; // see task #2401790
2096 constVal
= convIntToDouble(constVal
);
2097 emitLoadImm(m_as
, constVal
, dstReg
);
2099 // cvtsi2sd doesn't modify the high bits of its target, which can
2100 // cause false dependencies to prevent register renaming from kicking
2101 // in. Break the dependency chain by zeroing out the XMM reg.
2102 PhysReg srcReg
= m_regs
[src
].reg();
2103 PhysReg xmmReg
= dstReg
.isXMM() ? dstReg
: PhysReg(rCgXMM0
);
2104 m_as
.pxor_xmm_xmm(xmmReg
, xmmReg
);
2105 m_as
.cvtsi2sd_reg64_xmm(srcReg
, xmmReg
);
2106 zeroExtendIfBool(m_as
, src
, m_regs
[src
]);
2107 emitMovRegReg(m_as
, xmmReg
, dstReg
);
2111 void CodeGenerator::cgConvBoolToDbl(IRInstruction
* inst
) {
2112 emitConvBoolOrIntToDbl(inst
);
2115 void CodeGenerator::cgConvIntToDbl(IRInstruction
* inst
) {
2116 emitConvBoolOrIntToDbl(inst
);
2119 void CodeGenerator::cgConvBoolToInt(IRInstruction
* inst
) {
2120 SSATmp
* dst
= inst
->dst();
2121 auto dstReg
= m_regs
[dst
].reg();
2122 assert(dstReg
!= InvalidReg
);
2123 SSATmp
* src
= inst
->src(0);
2124 auto srcReg
= m_regs
[src
].reg();
2125 assert(src
->isConst() == (srcReg
== InvalidReg
));
2126 if (srcReg
== InvalidReg
) {
2127 int64_t constVal
= src
->getValRawInt();
2128 if (constVal
== 0) {
2129 m_as
.xor_reg64_reg64(dstReg
, dstReg
);
2131 m_as
.mov_imm64_reg(1, dstReg
);
2134 m_as
.movzbl(rbyte(srcReg
), r32(dstReg
));
2138 void CodeGenerator::cgConvBoolToStr(IRInstruction
* inst
) {
2139 SSATmp
* dst
= inst
->dst();
2140 auto dstReg
= m_regs
[dst
].reg();
2141 assert(dstReg
!= InvalidReg
);
2142 SSATmp
* src
= inst
->src(0);
2143 auto srcReg
= m_regs
[src
].reg();
2144 assert(src
->isConst() == (srcReg
== InvalidReg
));
2145 if (srcReg
== InvalidReg
) {
2146 auto constVal
= src
->getValBool();
2148 m_as
.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg
);
2150 m_as
.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), dstReg
);
2153 m_as
.testb(Reg8(int(srcReg
)), Reg8(int(srcReg
)));
2154 m_as
.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg
);
2155 m_as
.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), m_rScratch
);
2156 m_as
.cmov_reg64_reg64(CC_NZ
, m_rScratch
, dstReg
);
2160 void CodeGenerator::cgUnboxPtr(IRInstruction
* inst
) {
2161 SSATmp
* dst
= inst
->dst();
2162 SSATmp
* src
= inst
->src(0);
2164 auto srcReg
= m_regs
[src
].reg();
2165 auto dstReg
= m_regs
[dst
].reg();
2167 assert(srcReg
!= InvalidReg
);
2168 assert(dstReg
!= InvalidReg
);
2170 emitMovRegReg(m_as
, srcReg
, dstReg
);
2171 emitDerefIfVariant(m_as
, PhysReg(dstReg
));
2174 void CodeGenerator::cgUnbox(IRInstruction
* inst
) {
2175 SSATmp
* dst
= inst
->dst();
2176 SSATmp
* src
= inst
->src(0);
2177 auto dstValReg
= m_regs
[dst
].reg(0);
2178 auto dstTypeReg
= m_regs
[dst
].reg(1);
2179 auto srcValReg
= m_regs
[src
].reg(0);
2180 auto srcTypeReg
= m_regs
[src
].reg(1);
2182 assert(dstValReg
!= dstTypeReg
);
2183 assert(src
->type().equals(Type::Gen
));
2184 assert(dst
->type().notBoxed());
2186 emitCmpTVType(m_as
, HPHP::KindOfRef
, srcTypeReg
);
2187 ifThenElse(CC_E
, [&] {
2188 // srcTypeReg == KindOfRef; srcValReg is RefData*
2189 const size_t ref_tv_off
= RefData::tvOffset();
2190 if (dstValReg
!= srcValReg
) {
2191 emitLoadReg(m_as
, srcValReg
[ref_tv_off
+ TVOFF(m_data
)], dstValReg
);
2192 emitLoadTVType(m_as
, srcValReg
[ref_tv_off
+ TVOFF(m_type
)],
2195 emitLoadTVType(m_as
, srcValReg
[ref_tv_off
+ TVOFF(m_type
)],
2197 m_as
.loadq(srcValReg
[ref_tv_off
+ TVOFF(m_data
)], dstValReg
);
2200 // srcTypeReg != KindOfRef; copy src -> dst
2201 shuffle2(m_as
, srcValReg
, srcTypeReg
, dstValReg
, dstTypeReg
);
2205 void CodeGenerator::cgLdFuncCachedCommon(IRInstruction
* inst
) {
2206 SSATmp
* dst
= inst
->dst();
2207 SSATmp
* methodName
= inst
->src(0);
2209 const StringData
* name
= methodName
->getValStr();
2210 CacheHandle ch
= TargetCache::allocFixedFunction(name
);
2211 size_t funcCacheOff
= ch
+ offsetof(FixedFuncCache
, m_func
);
2213 auto dstReg
= m_regs
[dst
].reg();
2214 if (dstReg
== InvalidReg
) {
2215 // happens if LdFixedFunc and FCall not in same trace
2216 m_as
. cmpq(0, rVmTl
[funcCacheOff
]);
2218 m_as
. loadq (rVmTl
[funcCacheOff
], dstReg
);
2219 m_as
. testq (dstReg
, dstReg
);
2223 void CodeGenerator::cgLdFuncCached(IRInstruction
* inst
) {
2224 cgLdFuncCachedCommon(inst
);
2225 // jz off to the helper call in astubs
2226 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
2227 // this helper tries the autoload map, and fatals on failure
2228 cgCallNative(a
, inst
);
2232 void CodeGenerator::cgLdFuncCachedSafe(IRInstruction
* inst
) {
2233 cgLdFuncCachedCommon(inst
);
2234 if (Block
* taken
= inst
->taken()) {
2235 emitFwdJcc(m_as
, CC_Z
, taken
);
2239 void CodeGenerator::cgLdFunc(IRInstruction
* inst
) {
2240 SSATmp
* dst
= inst
->dst();
2241 SSATmp
* methodName
= inst
->src(0);
2243 TargetCache::CacheHandle ch
= TargetCache::FuncCache::alloc();
2244 // raises an error if function not found
2245 cgCallHelper(m_as
, (TCA
)FuncCache::lookup
, m_regs
[dst
].reg(),
2246 SyncOptions::kSyncPoint
,
2247 ArgGroup(m_regs
).imm(ch
).ssa(methodName
));
2250 static void emitLdObjClass(CodeGenerator::Asm
& a
,
2253 a
.loadq (objReg
[ObjectData::getVMClassOffset()], dstReg
);
2256 void CodeGenerator::cgLdObjClass(IRInstruction
* inst
) {
2257 auto dstReg
= m_regs
[inst
->dst()].reg();
2258 auto objReg
= m_regs
[inst
->src(0)].reg();
2260 emitLdObjClass(m_as
, objReg
, dstReg
);
2263 void CodeGenerator::cgLdObjMethod(IRInstruction
*inst
) {
2264 auto cls
= inst
->src(0);
2265 auto clsReg
= m_regs
[cls
].reg();
2266 auto name
= inst
->src(1);
2267 auto actRec
= inst
->src(2);
2268 auto actRecReg
= m_regs
[actRec
].reg();
2269 CacheHandle handle
= Transl::TargetCache::MethodCache::alloc();
2271 // lookup in the targetcache
2272 assert(MethodCache::kNumLines
== 1);
2274 MethodCache::Pair p
;
2275 static_assert(sizeof(p
.m_value
) == 8,
2276 "MethodCache::Pair::m_value assumed to be 8 bytes");
2277 static_assert(sizeof(p
.m_key
) == 8,
2278 "MethodCache::Pair::m_key assumed to be 8 bytes");
2281 // preload handle->m_value
2282 m_as
.loadq(rVmTl
[handle
+ offsetof(MethodCache::Pair
, m_value
)], m_rScratch
);
2283 m_as
.cmpq (rVmTl
[handle
+ offsetof(MethodCache::Pair
, m_key
)], clsReg
);
2284 ifThenElse(CC_E
, // if handle->key == cls
2285 [&] { // then actReg->m_func = handle->value
2286 m_as
.storeq(m_rScratch
, actRecReg
[AROFF(m_func
)]);
2288 [&] { // else call slow path helper
2289 cgCallHelper(m_as
, (TCA
)methodCacheSlowPath
, InvalidReg
,
2290 SyncOptions::kSyncPoint
,
2291 ArgGroup(m_regs
).addr(rVmTl
, handle
)
2298 void CodeGenerator::cgStRetVal(IRInstruction
* inst
) {
2299 auto const rFp
= m_regs
[inst
->src(0)].reg();
2300 auto* const val
= inst
->src(1);
2301 cgStore(rFp
, AROFF(m_r
), val
);
2304 void CodeGenerator::cgRetAdjustStack(IRInstruction
* inst
) {
2305 auto const rFp
= m_regs
[inst
->src(0)].reg();
2306 auto const dstSp
= m_regs
[inst
->dst()].reg();
2308 a
. lea (rFp
[AROFF(m_r
)], dstSp
);
2311 void CodeGenerator::cgLdRetAddr(IRInstruction
* inst
) {
2312 auto fpReg
= m_regs
[inst
->src(0)].reg(0);
2313 assert(fpReg
!= InvalidReg
);
2314 m_as
.push(fpReg
[AROFF(m_savedRip
)]);
2317 void checkFrame(ActRec
* fp
, Cell
* sp
, bool checkLocals
) {
2318 const Func
* func
= fp
->m_func
;
2319 if (fp
->hasVarEnv()) {
2320 assert(fp
->getVarEnv()->getCfp() == fp
);
2322 // TODO: validate this pointer from actrec
2323 int numLocals
= func
->numLocals();
2324 assert(sp
<= (Cell
*)fp
- func
->numSlotsInFrame()
2325 || func
->isGenerator());
2327 int numParams
= func
->numParams();
2328 for (int i
=0; i
< numLocals
; i
++) {
2329 if (i
>= numParams
&& func
->isGenerator() && i
< func
->numNamedLocals()) {
2332 assert(checkTv(frame_local(fp
, i
)));
2335 // We unfortunately can't do the same kind of check for the stack
2336 // without knowing about FPI regions, because it may contain
2340 void traceRet(ActRec
* fp
, Cell
* sp
, void* rip
) {
2341 if (rip
== TranslatorX64::Get()->getCallToExit()) {
2344 checkFrame(fp
, sp
, /*checkLocals*/ false);
2345 assert(sp
<= (Cell
*)fp
|| fp
->m_func
->isGenerator());
2346 // check return value if stack not empty
2347 if (sp
< (Cell
*)fp
) assertTv(sp
);
2350 void CodeGenerator::emitTraceRet(CodeGenerator::Asm
& a
) {
2351 // call to a trace function
2352 a
. movq (rVmFp
, rdi
);
2353 a
. movq (rVmSp
, rsi
);
2354 a
. loadq (*rsp
, rdx
); // return ip from native stack
2355 // do the call; may use a trampoline
2356 m_tx64
->emitCall(a
, TCA(traceRet
));
2359 void CodeGenerator::cgRetCtrl(IRInstruction
* inst
) {
2360 SSATmp
* sp
= inst
->src(0);
2361 SSATmp
* fp
= inst
->src(1);
2363 // Make sure rVmFp and rVmSp are set appropriately
2364 emitMovRegReg(m_as
, m_regs
[sp
].reg(), rVmSp
);
2365 emitMovRegReg(m_as
, m_regs
[fp
].reg(), rVmFp
);
2367 // Return control to caller
2368 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2372 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2377 void CodeGenerator::emitReqBindAddr(const Func
* func
,
2380 dest
= m_tx64
->emitServiceReq(SRFlags::None
,
2387 void CodeGenerator::cgJmpSwitchDest(IRInstruction
* inst
) {
2388 JmpSwitchData
* data
= inst
->extra
<JmpSwitchDest
>();
2389 SSATmp
* index
= inst
->src(0);
2390 auto indexReg
= m_regs
[index
].reg();
2392 if (!index
->isConst()) {
2393 if (data
->bounded
) {
2395 m_as
. subq(data
->base
, indexReg
);
2397 m_as
. cmpq(data
->cases
- 2, indexReg
);
2398 prepareForSmash(m_as
, kJmpccLen
);
2399 TCA def
= m_tx64
->emitServiceReq(REQ_BIND_JMPCC_SECOND
, 3,
2400 m_as
.code
.frontier
, data
->defaultOff
, CC_AE
);
2404 TCA
* table
= m_tx64
->allocData
<TCA
>(sizeof(TCA
), data
->cases
);
2405 TCA afterLea
= m_as
.code
.frontier
+ kLeaRipLen
;
2406 ptrdiff_t diff
= (TCA
)table
- afterLea
;
2407 assert(deltaFits(diff
, sz::dword
));
2408 m_as
. lea(rip
[diff
], m_rScratch
);
2409 assert(m_as
.code
.frontier
== afterLea
);
2410 m_as
. jmp(m_rScratch
[indexReg
*8]);
2412 for (int i
= 0; i
< data
->cases
; i
++) {
2413 emitReqBindAddr(data
->func
, table
[i
], data
->targets
[i
]);
2416 int64_t indexVal
= index
->getValInt();
2418 if (data
->bounded
) {
2419 indexVal
-= data
->base
;
2420 if (indexVal
>= data
->cases
- 2 || indexVal
< 0) {
2421 m_tx64
->emitBindJmp(m_as
, SrcKey(data
->func
, data
->defaultOff
));
2425 m_tx64
->emitBindJmp(m_as
, SrcKey(data
->func
, data
->targets
[indexVal
]));
2429 typedef FixedStringMap
<TCA
,true> SSwitchMap
;
2431 static TCA
sswitchHelperFast(const StringData
* val
,
2432 const SSwitchMap
* table
,
2434 TCA
* dest
= table
->find(val
);
2435 return dest
? *dest
: *def
;
2438 void CodeGenerator::cgLdSSwitchDestFast(IRInstruction
* inst
) {
2439 auto data
= inst
->extra
<LdSSwitchDestFast
>();
2441 auto table
= m_tx64
->allocData
<SSwitchMap
>(64);
2442 table
->init(data
->numCases
);
2443 for (int64_t i
= 0; i
< data
->numCases
; ++i
) {
2444 table
->add(data
->cases
[i
].str
, nullptr);
2445 TCA
* addr
= table
->find(data
->cases
[i
].str
);
2446 emitReqBindAddr(data
->func
, *addr
, data
->cases
[i
].dest
);
2448 TCA
* def
= m_tx64
->allocData
<TCA
>(sizeof(TCA
), 1);
2449 emitReqBindAddr(data
->func
, *def
, data
->defaultOff
);
2452 TCA(sswitchHelperFast
),
2454 SyncOptions::kNoSyncPoint
,
2461 static TCA
sswitchHelperSlow(TypedValue typedVal
,
2462 const StringData
** strs
,
2465 Cell
* cell
= tvToCell(&typedVal
);
2466 for (int i
= 0; i
< numStrs
; ++i
) {
2467 if (cellEqual(cell
, strs
[i
])) return jmptab
[i
];
2469 return jmptab
[numStrs
]; // default case
2472 void CodeGenerator::cgLdSSwitchDestSlow(IRInstruction
* inst
) {
2473 auto data
= inst
->extra
<LdSSwitchDestSlow
>();
2475 auto strtab
= m_tx64
->allocData
<const StringData
*>(
2476 sizeof(const StringData
*), data
->numCases
);
2477 auto jmptab
= m_tx64
->allocData
<TCA
>(sizeof(TCA
), data
->numCases
+ 1);
2478 for (int i
= 0; i
< data
->numCases
; ++i
) {
2479 strtab
[i
] = data
->cases
[i
].str
;
2480 emitReqBindAddr(data
->func
, jmptab
[i
], data
->cases
[i
].dest
);
2482 emitReqBindAddr(data
->func
, jmptab
[data
->numCases
], data
->defaultOff
);
2485 TCA(sswitchHelperSlow
),
2487 SyncOptions::kSyncPoint
,
2489 .typedValue(inst
->src(0))
2491 .imm(data
->numCases
)
2496 * It'd be nice not to have the cgMov here (and just copy propagate
2497 * the source or something), but for now we're keeping it allocated to
2498 * rVmFp so inlined calls to C++ helpers that use the rbp chain to
2499 * find the caller's ActRec will work correctly.
2501 * This instruction primarily exists to assist in optimizing away
2502 * unused activation records, so it's usually not going to happen
2505 void CodeGenerator::cgDefInlineFP(IRInstruction
* inst
) {
2506 auto const fp
= m_regs
[inst
->src(0)].reg();
2507 auto const fakeRet
= m_tx64
->getRetFromInlinedFrame();
2508 auto const retBCOff
= inst
->extra
<DefInlineFP
>()->retBCOff
;
2510 m_as
. storeq (fakeRet
, fp
[AROFF(m_savedRip
)]);
2511 m_as
. storel (retBCOff
, fp
[AROFF(m_soff
)]);
2516 void CodeGenerator::cgInlineReturn(IRInstruction
* inst
) {
2517 auto fpReg
= m_regs
[inst
->src(0)].reg();
2518 assert(fpReg
== rVmFp
);
2519 m_as
. loadq (fpReg
[AROFF(m_savedRbp
)], rVmFp
);
2522 void CodeGenerator::cgReDefSP(IRInstruction
* inst
) {
2523 // TODO(#2288359): this instruction won't be necessary (for
2524 // non-generator frames) when we don't track rVmSp independently
2525 // from rVmFp. In generator frames we'll have to track offsets from
2526 // a DefGeneratorSP or something similar.
2527 auto fp
= m_regs
[inst
->src(0)].reg();
2528 auto dst
= m_regs
[inst
->dst()].reg();
2529 auto off
= -inst
->extra
<ReDefSP
>()->offset
* sizeof(Cell
);
2530 emitLea(m_as
, fp
[off
], dst
);
2533 void CodeGenerator::cgStashGeneratorSP(IRInstruction
* inst
) {
2537 void CodeGenerator::cgReDefGeneratorSP(IRInstruction
* inst
) {
2541 void CodeGenerator::cgFreeActRec(IRInstruction
* inst
) {
2542 m_as
.loadq(m_regs
[inst
->src(0)].reg()[AROFF(m_savedRbp
)],
2543 m_regs
[inst
->dst()].reg());
2546 void CodeGenerator::cgSpill(IRInstruction
* inst
) {
2547 SSATmp
* dst
= inst
->dst();
2548 SSATmp
* src
= inst
->src(0);
2550 assert(dst
->numNeededRegs() == src
->numNeededRegs());
2551 for (int locIndex
= 0; locIndex
< m_regs
[src
].numAllocatedRegs();
2553 // We do not need to mask booleans, since the IR will reload the spill
2554 auto srcReg
= m_regs
[src
].reg(locIndex
);
2555 auto sinfo
= m_regs
[dst
].spillInfo(locIndex
);
2556 if (m_regs
[src
].isFullXMM()) {
2557 m_as
.movdqa(srcReg
, reg::rsp
[sinfo
.offset()]);
2559 int offset
= sinfo
.offset();
2560 if (locIndex
== 0 || packed_tv
|| src
->type().subtypeOf(Type::FuncCtx
)) {
2561 emitStoreReg(m_as
, srcReg
, reg::rsp
[offset
]);
2563 // Note that type field is shifted in memory
2564 assert(srcReg
.isGP());
2565 offset
+= TVOFF(m_type
) - (TVOFF(m_data
) + sizeof(Value
));
2566 emitStoreTVType(m_as
, srcReg
, reg::rsp
[offset
]);
2572 void CodeGenerator::cgReload(IRInstruction
* inst
) {
2573 SSATmp
* dst
= inst
->dst();
2574 SSATmp
* src
= inst
->src(0);
2576 assert(dst
->numNeededRegs() == src
->numNeededRegs());
2577 for (int locIndex
= 0; locIndex
< m_regs
[dst
].numAllocatedRegs();
2579 auto dstReg
= m_regs
[dst
].reg(locIndex
);
2580 auto sinfo
= m_regs
[src
].spillInfo(locIndex
);
2581 if (m_regs
[dst
].isFullXMM()) {
2582 assert(dstReg
.isXMM());
2583 m_as
.movdqa(reg::rsp
[sinfo
.offset()], dstReg
);
2585 int offset
= sinfo
.offset();
2586 if (locIndex
== 0 || packed_tv
|| src
->type().subtypeOf(Type::FuncCtx
)) {
2587 emitLoadReg(m_as
, reg::rsp
[offset
], dstReg
);
2589 // Note that type field is shifted in memory
2590 offset
+= TVOFF(m_type
) - (TVOFF(m_data
) + sizeof(Value
));
2591 assert(dstReg
.isGP());
2592 emitLoadTVType(m_as
, reg::rsp
[offset
], dstReg
);
2598 void CodeGenerator::cgStPropWork(IRInstruction
* inst
, bool genTypeStore
) {
2599 SSATmp
* obj
= inst
->src(0);
2600 SSATmp
* prop
= inst
->src(1);
2601 SSATmp
* src
= inst
->src(2);
2602 cgStore(m_regs
[obj
].reg(), prop
->getValInt(), src
, genTypeStore
);
2604 void CodeGenerator::cgStProp(IRInstruction
* inst
) {
2605 cgStPropWork(inst
, true);
2607 void CodeGenerator::cgStPropNT(IRInstruction
* inst
) {
2608 cgStPropWork(inst
, false);
2611 void CodeGenerator::cgStMemWork(IRInstruction
* inst
, bool genStoreType
) {
2612 SSATmp
* addr
= inst
->src(0);
2613 SSATmp
* offset
= inst
->src(1);
2614 SSATmp
* src
= inst
->src(2);
2615 cgStore(m_regs
[addr
].reg(), offset
->getValInt(), src
, genStoreType
);
2617 void CodeGenerator::cgStMem(IRInstruction
* inst
) {
2618 cgStMemWork(inst
, true);
2620 void CodeGenerator::cgStMemNT(IRInstruction
* inst
) {
2621 cgStMemWork(inst
, false);
2624 void CodeGenerator::cgStRefWork(IRInstruction
* inst
, bool genStoreType
) {
2625 auto destReg
= m_regs
[inst
->dst()].reg();
2626 auto addrReg
= m_regs
[inst
->src(0)].reg();
2627 SSATmp
* src
= inst
->src(1);
2628 always_assert(!m_regs
[src
].isFullXMM());
2629 cgStore(addrReg
, RefData::tvOffset(), src
, genStoreType
);
2630 if (destReg
!= InvalidReg
) emitMovRegReg(m_as
, addrReg
, destReg
);
2633 void CodeGenerator::cgStRef(IRInstruction
* inst
) {
2634 cgStRefWork(inst
, true);
2636 void CodeGenerator::cgStRefNT(IRInstruction
* inst
) {
2637 cgStRefWork(inst
, false);
2640 static int64_t localOffset(int64_t index
) {
2641 return -cellsToBytes(index
+ 1);
2644 static int64_t localOffset(SSATmp
* index
) {
2645 return localOffset(index
->getValInt());
2648 int CodeGenerator::iterOffset(SSATmp
* tmp
) {
2649 const Func
* func
= curFunc();
2650 int64_t index
= tmp
->getValInt();
2651 return -cellsToBytes(((index
+ 1) * kNumIterCells
+ func
->numLocals()));
2654 int CodeGenerator::iterOffset(uint32_t id
) {
2655 const Func
* func
= curFunc();
2656 return -cellsToBytes(((id
+ 1) * kNumIterCells
+ func
->numLocals()));
2659 void CodeGenerator::cgStLoc(IRInstruction
* inst
) {
2660 cgStore(m_regs
[inst
->src(0)].reg(),
2661 localOffset(inst
->extra
<StLoc
>()->locId
),
2663 true /* store type */);
2666 void CodeGenerator::cgStLocNT(IRInstruction
* inst
) {
2667 cgStore(m_regs
[inst
->src(0)].reg(),
2668 localOffset(inst
->extra
<StLocNT
>()->locId
),
2670 false /* store type */);
2673 void CodeGenerator::cgSyncABIRegs(IRInstruction
* inst
) {
2674 emitMovRegReg(m_as
, m_regs
[inst
->src(0)].reg(), rVmFp
);
2675 emitMovRegReg(m_as
, m_regs
[inst
->src(1)].reg(), rVmSp
);
2678 void CodeGenerator::cgReqBindJmp(IRInstruction
* inst
) {
2679 m_tx64
->emitBindJmp(
2681 SrcKey(curFunc(), inst
->extra
<ReqBindJmp
>()->offset
)
2685 static void emitExitNoIRStats(Asm
& a
,
2686 TranslatorX64
* tx64
,
2689 if (RuntimeOption::EnableInstructionCounts
||
2690 HPHP::Trace::moduleEnabled(HPHP::Trace::stats
, 3)) {
2692 Stats::opcodeToIRPreStatCounter(
2693 Op(*func
->unit()->at(dest
.offset()))),
2700 void CodeGenerator::cgReqBindJmpNoIR(IRInstruction
* inst
) {
2701 auto const dest
= SrcKey(curFunc(),
2702 inst
->extra
<ReqBindJmpNoIR
>()->offset
);
2703 emitExitNoIRStats(m_as
, m_tx64
, curFunc(), dest
);
2704 m_tx64
->emitBindJmp(m_as
, dest
, REQ_BIND_JMP_NO_IR
);
2707 void CodeGenerator::cgReqRetranslateNoIR(IRInstruction
* inst
) {
2708 auto const dest
= SrcKey(curFunc(),
2709 inst
->extra
<ReqRetranslateNoIR
>()->offset
);
2710 emitExitNoIRStats(m_as
, m_tx64
, curFunc(), dest
);
2711 m_tx64
->emitReqRetransNoIR(m_as
, dest
);
2714 void CodeGenerator::cgReqRetranslate(IRInstruction
* inst
) {
2715 auto const destSK
= SrcKey(curFunc(), m_curTrace
->bcOff());
2716 auto const destSR
= m_tx64
->getSrcRec(destSK
);
2717 m_tx64
->emitFallbackUncondJmp(m_as
, *destSR
);
2720 static void emitAssertFlagsNonNegative(CodeGenerator::Asm
& as
) {
2721 ifThen(as
, CC_NGE
, [&] { as
.ud2(); });
2724 static void emitAssertRefCount(CodeGenerator::Asm
& as
, PhysReg base
) {
2725 as
.cmpl(HPHP::RefCountStaticValue
, base
[FAST_REFCOUNT_OFFSET
]);
2726 ifThen(as
, CC_NBE
, [&] { as
.ud2(); });
2729 static void emitIncRef(CodeGenerator::Asm
& as
, PhysReg base
) {
2730 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2731 emitAssertRefCount(as
, base
);
2734 as
.incl(base
[FAST_REFCOUNT_OFFSET
]);
2735 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2736 // Assert that the ref count is greater than zero
2737 emitAssertFlagsNonNegative(as
);
2741 void CodeGenerator::cgIncRefWork(Type type
, SSATmp
* src
) {
2742 assert(type
.maybeCounted());
2743 auto increfMaybeStatic
= [&] {
2744 auto base
= m_regs
[src
].reg(0);
2745 if (!type
.needsStaticBitCheck()) {
2746 emitIncRef(m_as
, base
);
2748 m_as
.cmpl(RefCountStaticValue
, base
[FAST_REFCOUNT_OFFSET
]);
2749 ifThen(m_as
, CC_NE
, [&] { emitIncRef(m_as
, base
); });
2753 if (type
.isKnownDataType()) {
2754 assert(IS_REFCOUNTED_TYPE(type
.toDataType()));
2755 increfMaybeStatic();
2757 emitCmpTVType(m_as
, KindOfRefCountThreshold
, m_regs
[src
].reg(1));
2758 ifThen(m_as
, CC_NLE
, [&] { increfMaybeStatic(); });
2762 void CodeGenerator::cgIncRef(IRInstruction
* inst
) {
2763 SSATmp
* dst
= inst
->dst();
2764 SSATmp
* src
= inst
->src(0);
2765 Type type
= src
->type();
2767 cgIncRefWork(type
, src
);
2768 shuffle2(m_as
, m_regs
[src
].reg(0), m_regs
[src
].reg(1),
2769 m_regs
[dst
].reg(0), m_regs
[dst
].reg(1));
2772 void CodeGenerator::cgDecRefStack(IRInstruction
* inst
) {
2773 cgDecRefMem(inst
->typeParam(),
2774 m_regs
[inst
->src(0)].reg(),
2775 cellsToBytes(inst
->extra
<DecRefStack
>()->offset
),
2779 void CodeGenerator::cgDecRefThis(IRInstruction
* inst
) {
2780 SSATmp
* fp
= inst
->src(0);
2781 Block
* exit
= inst
->taken();
2782 auto fpReg
= m_regs
[fp
].reg();
2783 auto scratchReg
= m_rScratch
;
2785 // Load AR->m_this into m_rScratch
2786 m_as
.loadq(fpReg
[AROFF(m_this
)], scratchReg
);
2788 auto decrefIfAvailable
= [&] {
2789 // Check if this is available and we're not in a static context instead
2790 m_as
.testb(1, rbyte(scratchReg
));
2791 ifThen(m_as
, CC_Z
, [&] {
2796 true /* genZeroCheck */
2801 if (curFunc()->isPseudoMain()) {
2802 // In pseudo-mains, emit check for presence of m_this
2803 m_as
.testq(scratchReg
, scratchReg
);
2804 ifThen(m_as
, CC_NZ
, [&] { decrefIfAvailable(); });
2806 decrefIfAvailable();
2810 void CodeGenerator::cgDecRefLoc(IRInstruction
* inst
) {
2811 cgDecRefMem(inst
->typeParam(),
2812 m_regs
[inst
->src(0)].reg(),
2813 localOffset(inst
->extra
<DecRefLoc
>()->locId
),
2817 void CodeGenerator::cgGenericRetDecRefs(IRInstruction
* inst
) {
2818 auto const rFp
= m_regs
[inst
->src(0)].reg();
2819 auto const numLocals
= inst
->src(1)->getValInt();
2820 auto const rDest
= m_regs
[inst
->dst()].reg();
2823 assert(rFp
== rVmFp
&&
2824 "free locals helper assumes the frame pointer is rVmFp");
2825 assert(rDest
== rVmSp
&&
2826 "free locals helper adjusts rVmSp, which must be our dst reg");
2828 if (numLocals
== 0) return;
2830 // The helpers called below use a special ABI, in which r15 is not saved.
2831 // So save r15 on the stack if it's live.
2832 bool saveR15
= m_state
.liveRegs
[inst
].contains(r15
);
2834 int stackAdjust
= 8;
2840 auto const target
= numLocals
> kNumFreeLocalsHelpers
2841 ? m_tx64
->m_freeManyLocalsHelper
2842 : m_tx64
->m_freeLocalsHelpers
[numLocals
- 1];
2844 a
.subq(stackAdjust
, rsp
); // For parity; callee does retq $0x8.
2845 a
.lea(rFp
[-numLocals
* sizeof(TypedValue
)], rDest
);
2856 tv_release_generic(TypedValue
* tv
) {
2857 assert(Transl::tx64
->stateIsDirty());
2858 assert(tv
->m_type
>= KindOfString
&& tv
->m_type
<= KindOfRef
);
2859 g_destructors
[typeToDestrIndex(tv
->m_type
)](tv
->m_data
.pref
);
2863 tv_release_typed(RefData
* pv
, DataType dt
) {
2864 assert(Transl::tx64
->stateIsDirty());
2865 assert(dt
>= KindOfString
&& dt
<= KindOfRef
);
2866 g_destructors
[typeToDestrIndex(dt
)](pv
);
2869 Address
CodeGenerator::getDtorGeneric() {
2870 return (Address
)tv_release_generic
;
2873 Address
CodeGenerator::getDtorTyped() {
2874 return (Address
)tv_release_typed
;
2878 // This method generates code that checks the static bit and jumps if the bit
2879 // is set. If regIsCount is true, reg contains the _count field. Otherwise,
2880 // it's assumed to contain m_data field.
2882 // Return value: the address to be patched with the address to jump to in case
2883 // the static bit is set. If the check is unnecessary, this method retuns NULL.
2884 Address
CodeGenerator::cgCheckStaticBit(Type type
,
2887 if (!type
.needsStaticBitCheck()) return NULL
;
2890 // reg has the _count value
2891 m_as
.cmp_imm32_reg32(RefCountStaticValue
, reg
);
2893 // reg has the data pointer
2894 m_as
.cmp_imm32_disp_reg32(RefCountStaticValue
, FAST_REFCOUNT_OFFSET
, reg
);
2897 Address addrToPatch
= m_as
.code
.frontier
;
2898 m_as
.jcc8(CC_E
, addrToPatch
);
2904 // Using the given dataReg, this method generates code that checks the static
2905 // bit out of dataReg, and emits a DecRef if needed.
2906 // NOTE: the flags are left with the result of the DecRef's subtraction,
2907 // which can then be tested immediately after this.
2909 // Return value: the address to be patched if a RefCountedStaticValue check is
2910 // emitted; NULL otherwise.
2912 Address
CodeGenerator::cgCheckStaticBitAndDecRef(Type type
,
2915 assert(type
.maybeCounted());
2917 Address patchStaticCheck
= nullptr;
2918 const auto scratchReg
= m_rScratch
;
2920 bool canUseScratch
= dataReg
!= scratchReg
;
2922 // TODO: run experiments to check whether the 'if' code sequence
2923 // is any better than the 'else' branch below; otherwise, always
2924 // use the 'else' code sequence
2925 if (type
.needsStaticBitCheck() && canUseScratch
) {
2926 // If we need to check for static value, then load the _count into a
2927 // register to avoid doing two loads. The generated sequence is:
2929 // scratchReg = [dataReg + offset(_count)]
2930 // if scratchReg == RefCountStaticValue then skip DecRef
2931 // scratchReg = scratchReg - 1
2932 // ( if exit != NULL, emit:
2935 // [dataReg + offset(_count)] = scratchReg
2937 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2938 emitAssertRefCount(m_as
, dataReg
);
2940 // Load _count in scratchReg
2941 m_as
.loadl(dataReg
[FAST_REFCOUNT_OFFSET
], r32(scratchReg
));
2943 // Check for RefCountStaticValue
2944 patchStaticCheck
= cgCheckStaticBit(type
, scratchReg
,
2945 true /* reg has _count */);
2947 // Decrement count and store it back in memory.
2948 // If there's an exit, emit jump to it when _count would get down to 0
2949 m_as
.decq(scratchReg
);
2951 emitFwdJcc(CC_E
, exit
);
2953 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2954 // Assert that the ref count is greater than zero
2955 emitAssertFlagsNonNegative(m_as
);
2957 m_as
.store_reg32_disp_reg64(scratchReg
, FAST_REFCOUNT_OFFSET
, dataReg
);
2960 // Can't use scratch reg, so emit code that operates directly in
2961 // memory. Compared to the sequence above, this will result in one
2962 // extra load, but it has the advantage of producing a instruction
2965 // ( if needStaticBitCheck, emit :
2966 // cmp [dataReg + offset(_count)], RefCountStaticValue
2967 // je LabelAfterDecRef
2969 // ( if exit != NULL, emit:
2970 // cmp [dataReg + offset(_count)], 1
2973 // sub [dataReg + offset(_count)], 1
2975 // If necessary, check for RefCountStaticValue
2976 patchStaticCheck
= cgCheckStaticBit(type
, dataReg
,
2977 false /* passing dataReg */);
2979 // If there's an exit, emit jump to it if _count would get down to 0
2981 m_as
.cmp_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET
, dataReg
);
2982 emitFwdJcc(CC_E
, exit
);
2984 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2985 emitAssertRefCount(m_as
, dataReg
);
2989 m_as
.decl(dataReg
[FAST_REFCOUNT_OFFSET
]);
2991 if (RuntimeOption::EvalHHIRGenerateAsserts
) {
2992 // Assert that the ref count is not less than zero
2993 emitAssertFlagsNonNegative(m_as
);
2997 return patchStaticCheck
;
3002 // Returns the address to be patched with the address to jump to in case
3003 // the type is not ref-counted.
3005 Address
CodeGenerator::cgCheckRefCountedType(PhysReg typeReg
) {
3006 emitCmpTVType(m_as
, KindOfRefCountThreshold
, typeReg
);
3007 Address addrToPatch
= m_as
.code
.frontier
;
3008 m_as
.jcc8(CC_LE
, addrToPatch
);
3012 Address
CodeGenerator::cgCheckRefCountedType(PhysReg baseReg
, int64_t offset
) {
3013 emitCmpTVType(m_as
, KindOfRefCountThreshold
, baseReg
[offset
+ TVOFF(m_type
)]);
3014 Address addrToPatch
= m_as
.code
.frontier
;
3015 m_as
.jcc8(CC_LE
, addrToPatch
);
3020 // Generates dec-ref of a typed value with statically known type.
3022 void CodeGenerator::cgDecRefStaticType(Type type
,
3025 bool genZeroCheck
) {
3026 assert(type
!= Type::Cell
&& type
!= Type::Gen
);
3027 assert(type
.isKnownDataType());
3029 if (type
.notCounted()) return;
3031 // Check for RefCountStaticValue if needed, do the actual DecRef,
3032 // and leave flags set based on the subtract result, which is
3034 Address patchStaticCheck
;
3036 patchStaticCheck
= cgCheckStaticBitAndDecRef(type
, dataReg
, exit
);
3038 // Set exit as NULL so that the code doesn't jump to error checking.
3039 patchStaticCheck
= cgCheckStaticBitAndDecRef(type
, dataReg
, nullptr);
3042 // If not exiting on count down to zero, emit the zero-check and
3044 if (genZeroCheck
&& exit
== nullptr) {
3045 // Emit jump to m_astubs (to call release) if count got down to zero
3046 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
3047 // Emit the call to release in m_astubs
3048 cgCallHelper(a
, m_tx64
->getDtorCall(type
.toDataType()),
3049 InvalidReg
, InvalidReg
, SyncOptions::kSyncPoint
,
3050 ArgGroup(m_regs
).reg(dataReg
));
3053 if (patchStaticCheck
) {
3054 m_as
.patchJcc8(patchStaticCheck
, m_as
.code
.frontier
);
3059 // Generates dec-ref of a typed value with dynamic (statically unknown) type,
3060 // when the type is stored in typeReg.
3062 void CodeGenerator::cgDecRefDynamicType(PhysReg typeReg
,
3065 bool genZeroCheck
) {
3066 // Emit check for ref-counted type
3067 Address patchTypeCheck
= cgCheckRefCountedType(typeReg
);
3069 // Emit check for RefCountStaticValue and the actual DecRef
3070 Address patchStaticCheck
;
3072 patchStaticCheck
= cgCheckStaticBitAndDecRef(Type::Cell
, dataReg
, exit
);
3074 patchStaticCheck
= cgCheckStaticBitAndDecRef(Type::Cell
, dataReg
, nullptr);
3077 // If not exiting on count down to zero, emit the zero-check and release call
3078 if (genZeroCheck
&& exit
== nullptr) {
3079 // Emit jump to m_astubs (to call release) if count got down to zero
3080 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
3081 // Emit call to release in m_astubs
3082 cgCallHelper(a
, getDtorTyped(), InvalidReg
, SyncOptions::kSyncPoint
,
3083 ArgGroup(m_regs
).reg(dataReg
).reg(typeReg
));
3086 // Patch checks to jump around the DecRef
3087 if (patchTypeCheck
) m_as
.patchJcc8(patchTypeCheck
, m_as
.code
.frontier
);
3088 if (patchStaticCheck
) m_as
.patchJcc8(patchStaticCheck
, m_as
.code
.frontier
);
3092 // Generates dec-ref of a typed value with dynamic (statically
3093 // unknown) type, when all we have is the baseReg and offset of
3094 // the typed value. This method assumes that baseReg is not the
3095 // scratch register.
3097 void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg
,
3100 auto scratchReg
= m_rScratch
;
3102 assert(baseReg
!= scratchReg
);
3104 // Emit check for ref-counted type
3105 Address patchTypeCheck
= cgCheckRefCountedType(baseReg
, offset
);
3106 if (exit
== nullptr && RuntimeOption::EvalHHIRGenericDtorHelper
) {
3108 // This PhysRegSaverStub saves rdi redundantly if
3109 // !liveRegs[m_curInst].contains(rdi), but its
3110 // necessary to maintain stack alignment. We can do better
3111 // by making the helpers adjust the stack for us in the cold
3112 // path, which calls the destructor.
3113 PhysRegSaverStub
regSaver(m_as
, RegSet(rdi
));
3116 * rVmSp is ok here because this is part of the special
3117 * ABI to m_irPopRHelper. We're not using a secret dependency
3118 * on the frame or stack---we're only going to use that ABI if
3119 * we happen to have that register allocated for baseReg.
3121 if (offset
== 0 && baseReg
== rVmSp
) {
3122 // Decref'ing top of vm stack, very likely a popR
3123 m_tx64
->emitCall(m_as
, m_tx64
->m_irPopRHelper
);
3125 if (baseReg
== rsp
) {
3126 // Because we just pushed %rdi, %rsp is 8 bytes below where
3127 // offset is expecting it to be.
3128 offset
+= sizeof(int64_t);
3130 m_as
.lea(baseReg
[offset
], rdi
);
3131 m_tx64
->emitCall(m_as
, m_tx64
->m_dtorGenericStub
);
3133 recordSyncPoint(m_as
);
3135 if (patchTypeCheck
) {
3136 m_as
.patchJcc8(patchTypeCheck
, m_as
.code
.frontier
);
3140 // Load m_data into the scratch reg
3141 m_as
.loadq(baseReg
[offset
+ TVOFF(m_data
)], scratchReg
);
3143 // Emit check for RefCountStaticValue and the actual DecRef
3144 Address patchStaticCheck
= cgCheckStaticBitAndDecRef(Type::Cell
, scratchReg
,
3147 // If not exiting on count down to zero, emit the zero-check and release call
3148 if (exit
== nullptr) {
3149 // Emit jump to m_astubs (to call release) if count got down to zero
3150 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
3151 // Emit call to release in m_astubs
3152 a
.lea(baseReg
[offset
], scratchReg
);
3153 cgCallHelper(a
, getDtorGeneric(), InvalidReg
, SyncOptions::kSyncPoint
,
3154 ArgGroup(m_regs
).reg(scratchReg
));
3158 // Patch checks to jump around the DecRef
3159 if (patchTypeCheck
) m_as
.patchJcc8(patchTypeCheck
, m_as
.code
.frontier
);
3160 if (patchStaticCheck
) m_as
.patchJcc8(patchStaticCheck
, m_as
.code
.frontier
);
3164 // Generates the dec-ref of a typed value in memory address [baseReg + offset].
3165 // This handles cases where type is either static or dynamic.
3167 void CodeGenerator::cgDecRefMem(Type type
,
3171 auto scratchReg
= m_rScratch
;
3172 assert(baseReg
!= scratchReg
);
3174 if (type
.needsReg()) {
3175 // The type is dynamic, but we don't have two registers available
3176 // to load the type and the data.
3177 cgDecRefDynamicTypeMem(baseReg
, offset
, exit
);
3178 } else if (type
.maybeCounted()) {
3179 m_as
.loadq(baseReg
[offset
+ TVOFF(m_data
)], scratchReg
);
3180 cgDecRefStaticType(type
, scratchReg
, exit
, true);
3184 void CodeGenerator::cgDecRefMem(IRInstruction
* inst
) {
3185 assert(inst
->src(0)->type().isPtr());
3186 cgDecRefMem(inst
->typeParam(),
3187 m_regs
[inst
->src(0)].reg(),
3188 inst
->src(1)->getValInt(),
3192 void CodeGenerator::cgDecRefWork(IRInstruction
* inst
, bool genZeroCheck
) {
3193 SSATmp
* src
= inst
->src(0);
3194 if (!isRefCounted(src
)) return;
3195 Block
* exit
= inst
->taken();
3196 Type type
= src
->type();
3197 if (type
.isKnownDataType()) {
3198 cgDecRefStaticType(type
, m_regs
[src
].reg(), exit
, genZeroCheck
);
3200 cgDecRefDynamicType(m_regs
[src
].reg(1),
3207 void CodeGenerator::cgDecRef(IRInstruction
*inst
) {
3208 // DecRef may bring the count to zero, and run the destructor.
3209 // Generate code for this.
3210 assert(!inst
->taken());
3211 cgDecRefWork(inst
, true);
3214 void CodeGenerator::cgDecRefNZ(IRInstruction
* inst
) {
3215 // DecRefNZ cannot bring the count to zero.
3216 // Therefore, we don't generate zero-checking code.
3217 assert(!inst
->taken());
3218 cgDecRefWork(inst
, false);
3221 void CodeGenerator::cgDecRefNZOrBranch(IRInstruction
* inst
) {
3222 assert(inst
->taken());
3223 cgDecRefWork(inst
, true);
3226 void CodeGenerator::cgCufIterSpillFrame(IRInstruction
* inst
) {
3227 auto const sp
= inst
->src(0);
3228 auto const fp
= inst
->src(1);
3229 auto const nArgs
= inst
->extra
<CufIterSpillFrame
>()->args
;
3230 auto const iterId
= inst
->extra
<CufIterSpillFrame
>()->iterId
;
3231 auto const itOff
= iterOffset(iterId
);
3233 const int64_t spOffset
= -kNumActRecCells
* sizeof(Cell
);
3234 auto spReg
= m_regs
[sp
].reg();
3235 auto fpReg
= m_regs
[fp
].reg();
3237 m_as
.loadq (fpReg
[itOff
+ CufIter::funcOff()], m_rScratch
);
3238 m_as
.storeq (m_rScratch
, spReg
[spOffset
+ int(AROFF(m_func
))]);
3240 m_as
.loadq (fpReg
[itOff
+ CufIter::ctxOff()], m_rScratch
);
3241 m_as
.storeq (m_rScratch
, spReg
[spOffset
+ int(AROFF(m_this
))]);
3243 m_as
.shrq (1, m_rScratch
);
3244 ifThen(m_as
, CC_NBE
, [this] {
3245 m_as
.shlq(1, m_rScratch
);
3246 emitIncRef(m_as
, m_rScratch
);
3248 m_as
.loadq (fpReg
[itOff
+ CufIter::nameOff()], m_rScratch
);
3249 m_as
.testq (m_rScratch
, m_rScratch
);
3250 ifThen(m_as
, CC_NZ
, [this] {
3251 m_as
.cmpl(RefCountStaticValue
, m_rScratch
[FAST_REFCOUNT_OFFSET
]);
3252 ifThen(m_as
, CC_NE
, [&] { emitIncRef(m_as
, m_rScratch
); });
3253 m_as
.orq (ActRec::kInvNameBit
, m_rScratch
);
3255 m_as
.storeq (m_rScratch
, spReg
[spOffset
+ int(AROFF(m_invName
))]);
3256 m_as
.storeq (fpReg
, spReg
[spOffset
+ int(AROFF(m_savedRbp
))]);
3257 m_as
.storel (nArgs
, spReg
[spOffset
+ int(AROFF(m_numArgsAndCtorFlag
))]);
3260 m_regs
[inst
->dst()].reg(),
3264 void CodeGenerator::cgSpillFrame(IRInstruction
* inst
) {
3265 auto const sp
= inst
->src(0);
3266 auto const fp
= inst
->src(1);
3267 auto const func
= inst
->src(2);
3268 auto const objOrCls
= inst
->src(3);
3269 auto const magicName
= inst
->extra
<SpillFrame
>()->invName
;
3270 auto const nArgs
= inst
->extra
<SpillFrame
>()->numArgs
;
3272 const int64_t spOffset
= -kNumActRecCells
* sizeof(Cell
);
3274 DEBUG_ONLY
bool setThis
= true;
3276 auto spReg
= m_regs
[sp
].reg();
3278 if (objOrCls
->isA(Type::Cls
)) {
3280 if (objOrCls
->isConst()) {
3281 m_as
.store_imm64_disp_reg64(uintptr_t(objOrCls
->getValClass()) | 1,
3282 spOffset
+ int(AROFF(m_this
)),
3285 Reg64 clsPtrReg
= m_regs
[objOrCls
].reg();
3286 m_as
.movq (clsPtrReg
, m_rScratch
);
3287 m_as
.orq (1, m_rScratch
);
3288 m_as
.storeq(m_rScratch
, spReg
[spOffset
+ int(AROFF(m_this
))]);
3290 } else if (objOrCls
->isA(Type::Obj
)) {
3291 // store this pointer
3292 m_as
.store_reg64_disp_reg64(m_regs
[objOrCls
].reg(),
3293 spOffset
+ int(AROFF(m_this
)),
3295 } else if (objOrCls
->isA(Type::Ctx
)) {
3296 // Stores either a this pointer or a Cctx -- statically unknown.
3297 Reg64 objOrClsPtrReg
= m_regs
[objOrCls
].reg();
3298 m_as
.storeq(objOrClsPtrReg
, spReg
[spOffset
+ int(AROFF(m_this
))]);
3300 assert(objOrCls
->isA(Type::InitNull
));
3301 // no obj or class; this happens in FPushFunc
3302 int offset_m_this
= spOffset
+ int(AROFF(m_this
));
3303 // When func is either Type::FuncCls or Type::FuncCtx,
3304 // m_this/m_cls will be initialized below
3305 if (!func
->isConst() && (func
->isA(Type::FuncCtx
))) {
3306 // m_this is unioned with m_cls and will be initialized below
3309 m_as
.store_imm64_disp_reg64(0, offset_m_this
, spReg
);
3312 // actRec->m_invName
3313 // ActRec::m_invName is encoded as a pointer with bit kInvNameBit
3314 // set to distinguish it from m_varEnv and m_extrArgs
3315 uintptr_t invName
= !magicName
3317 : reinterpret_cast<uintptr_t>(magicName
) | ActRec::kInvNameBit
;
3318 m_as
.store_imm64_disp_reg64(invName
,
3319 spOffset
+ int(AROFF(m_invName
)),
3321 // actRec->m_func and possibly actRec->m_cls
3322 // Note m_cls is unioned with m_this and may overwrite previous value
3323 if (func
->type().isNull()) {
3324 assert(func
->isConst());
3325 } else if (func
->isConst()) {
3326 const Func
* f
= func
->getValFunc();
3327 m_as
. mov_imm64_reg((uint64_t)f
, m_rScratch
);
3328 m_as
.store_reg64_disp_reg64(m_rScratch
,
3329 spOffset
+ int(AROFF(m_func
)),
3331 if (func
->isA(Type::FuncCtx
)) {
3332 // Fill in m_cls if provided with both func* and class*
3333 CG_PUNT(cgAllocActRec
);
3336 int offset_m_func
= spOffset
+ int(AROFF(m_func
));
3337 m_as
.store_reg64_disp_reg64(m_regs
[func
].reg(0),
3340 if (func
->isA(Type::FuncCtx
)) {
3341 int offset_m_cls
= spOffset
+ int(AROFF(m_cls
));
3342 m_as
.store_reg64_disp_reg64(m_regs
[func
].reg(1),
3345 setThis
= true; /* m_this and m_cls are in a union */
3349 // actRec->m_savedRbp
3350 m_as
.store_reg64_disp_reg64(m_regs
[fp
].reg(),
3351 spOffset
+ int(AROFF(m_savedRbp
)),
3354 // actRec->m_numArgsAndCtorFlag
3355 m_as
.store_imm32_disp_reg(nArgs
,
3356 spOffset
+ int(AROFF(m_numArgsAndCtorFlag
)),
3360 m_regs
[inst
->dst()].reg(),
3364 const Func
* loadClassCtor(Class
* cls
) {
3365 const Func
* f
= cls
->getCtor();
3366 if (UNLIKELY(!(f
->attrs() & AttrPublic
))) {
3368 UNUSED
MethodLookup::LookupResult res
=
3369 g_vmContext
->lookupCtorMethod(f
, cls
, true /*raise*/);
3370 assert(res
== MethodLookup::LookupResult::MethodFoundWithThis
);
3375 Instance
* createClHelper(Class
* cls
, int numArgs
, ActRec
* ar
, TypedValue
* sp
) {
3376 Instance
* newObj
= newInstance(cls
);
3377 newObj
->incRefCount();
3378 return static_cast<c_Closure
*>(newObj
)->init(numArgs
, ar
, sp
);
3381 void CodeGenerator::cgAllocObjFast(IRInstruction
* inst
) {
3382 const Class
* cls
= inst
->src(0)->getValClass();
3383 auto dstReg
= m_regs
[inst
->dst()].reg();
3385 // First, make sure our property init vectors are all set up
3386 bool props
= cls
->pinitVec().size() > 0;
3387 bool sprops
= cls
->numStaticProperties() > 0;
3388 assert((props
|| sprops
) == cls
->needInitialization());
3389 if (cls
->needInitialization()) {
3391 cls
->initPropHandle();
3392 m_as
.testq(-1, rVmTl
[cls
->propHandle()]);
3393 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
3395 (TCA
)getMethodPtr(&Class::initProps
),
3397 SyncOptions::kSyncPoint
,
3398 ArgGroup(m_regs
).imm((uint64_t)cls
));
3402 cls
->initSPropHandle();
3403 m_as
.testq(-1, rVmTl
[cls
->sPropHandle()]);
3404 unlikelyIfBlock(CC_Z
, [&] (Asm
& a
) {
3406 (TCA
)getMethodPtr(&Class::initSProps
),
3408 SyncOptions::kSyncPoint
,
3409 ArgGroup(m_regs
).imm((uint64_t)cls
));
3414 // Next, allocate the object
3415 if (cls
->instanceCtor()) {
3417 (TCA
)cls
->instanceCtor(),
3419 SyncOptions::kSyncPoint
,
3420 ArgGroup(m_regs
).imm((uint64_t)cls
));
3422 size_t size
= Instance::sizeForNProps(cls
->numDeclProperties());
3423 int allocator
= object_alloc_size_to_index(size
);
3424 assert(allocator
!= -1);
3426 (TCA
)getMethodPtr(&Instance::newInstanceRaw
),
3428 SyncOptions::kSyncPoint
,
3429 ArgGroup(m_regs
).imm((uint64_t)cls
).imm(allocator
));
3432 // Set the attributes, if any
3433 int odAttrs
= cls
->getODAttrs();
3435 // o_attribute is 16 bits but the fact that we're or-ing a mask makes it ok
3436 assert(!(odAttrs
& 0xffff0000));
3437 m_as
.orq(odAttrs
, dstReg
[ObjectData::attributeOff()]);
3440 // Initialize the properties
3441 size_t nProps
= cls
->numDeclProperties();
3444 m_as
.subq(8, reg::rsp
);
3445 if (cls
->pinitVec().size() == 0) {
3446 // Fast case: copy from a known address in the Class
3447 ArgGroup args
= ArgGroup(m_regs
)
3448 .addr(dstReg
, sizeof(ObjectData
) + cls
->builtinPropSize())
3449 .imm(int64_t(&cls
->declPropInit()[0]))
3450 .imm(cellsToBytes(nProps
));
3454 SyncOptions::kNoSyncPoint
,
3457 // Slower case: we have to load the src address from the targetcache
3458 auto rPropData
= m_rScratch
;
3459 // Load the Class's propInitVec from the targetcache
3460 m_as
.loadq(rVmTl
[cls
->propHandle()], rPropData
);
3461 // propData holds the PropInitVec. We want &(*propData)[0]
3462 m_as
.loadq(rPropData
[Class::PropInitVec::dataOff()], rPropData
);
3463 if (!cls
->hasDeepInitProps()) {
3464 ArgGroup args
= ArgGroup(m_regs
)
3465 .addr(dstReg
, sizeof(ObjectData
) + cls
->builtinPropSize())
3467 .imm(cellsToBytes(nProps
));
3471 SyncOptions::kNoSyncPoint
,
3474 ArgGroup args
= ArgGroup(m_regs
)
3475 .addr(dstReg
, sizeof(ObjectData
) + cls
->builtinPropSize())
3479 (TCA
)deepInitHelper
,
3481 SyncOptions::kNoSyncPoint
,
3485 m_as
.addq(8, reg::rsp
);
3488 if (cls
->callsCustomInstanceInit()) {
3489 // callCustomInstanceInit returns the instance in rax
3491 (TCA
)getMethodPtr(&Instance::callCustomInstanceInit
),
3493 SyncOptions::kSyncPoint
,
3494 ArgGroup(m_regs
).reg(dstReg
));
3498 void CodeGenerator::cgCallArray(IRInstruction
* inst
) {
3499 Offset pc
= inst
->extra
<CallArray
>()->pc
;
3500 Offset after
= inst
->extra
<CallArray
>()->after
;
3502 ArgGroup
args(m_regs
);
3503 args
.imm(pc
).imm(after
);
3505 // fCallArrayHelper makes the actual call by smashing its return address.
3506 cgCallHelper(m_as
, (TCA
)TranslatorX64::fCallArrayHelper
,
3507 nullptr, SyncOptions::kSyncPoint
, args
);
3510 void CodeGenerator::cgCall(IRInstruction
* inst
) {
3511 SSATmp
* actRec
= inst
->src(0);
3512 SSATmp
* returnBcOffset
= inst
->src(1);
3513 SSATmp
* func
= inst
->src(2);
3514 SrcRange args
= inst
->srcs().subpiece(3);
3515 int32_t numArgs
= args
.size();
3517 auto spReg
= m_regs
[actRec
].reg();
3518 // put all outgoing arguments onto the VM stack
3519 int64_t adjustment
= (-(int64_t)numArgs
) * sizeof(Cell
);
3520 for (int32_t i
= 0; i
< numArgs
; i
++) {
3521 // Type::None here means that the simplifier proved that the value
3522 // matches the value already in memory, thus the store is redundant.
3523 if (args
[i
]->type() != Type::None
) {
3524 cgStore(spReg
, -(i
+ 1) * sizeof(Cell
), args
[i
]);
3527 // store the return bytecode offset into the outgoing actrec
3528 uint64_t returnBc
= returnBcOffset
->getValInt();
3529 m_as
.store_imm32_disp_reg(returnBc
, AROFF(m_soff
), spReg
);
3530 if (adjustment
!= 0) {
3531 m_as
.add_imm32_reg64(adjustment
, spReg
);
3534 assert(m_curInst
->marker().valid());
3535 SrcKey srcKey
= SrcKey(m_curInst
->marker().func
, m_curInst
->marker().bcOff
);
3536 bool isImmutable
= (func
->isConst() && !func
->type().isNull());
3537 const Func
* funcd
= isImmutable
? func
->getValFunc() : nullptr;
3538 assert(&m_as
== &m_tx64
->getAsm());
3539 int32_t adjust
= m_tx64
->emitBindCall(srcKey
, funcd
, numArgs
);
3541 m_as
.addq (adjust
, rVmSp
);
3545 void CodeGenerator::cgCastStk(IRInstruction
*inst
) {
3546 Type type
= inst
->typeParam();
3547 SSATmp
* sp
= inst
->src(0);
3548 uint32_t offset
= inst
->extra
<CastStk
>()->offset
;
3549 PhysReg spReg
= m_regs
[sp
].reg();
3551 ArgGroup
args(m_regs
);
3552 args
.addr(spReg
, cellsToBytes(offset
));
3555 if (type
.subtypeOf(Type::Bool
)) {
3556 tvCastHelper
= (TCA
)tvCastToBooleanInPlace
;
3557 } else if (type
.subtypeOf(Type::Int
)) {
3558 // if casting to integer, pass 10 as the base for the conversion
3560 tvCastHelper
= (TCA
)tvCastToInt64InPlace
;
3561 } else if (type
.subtypeOf(Type::Dbl
)) {
3562 tvCastHelper
= (TCA
)tvCastToDoubleInPlace
;
3563 } else if (type
.subtypeOf(Type::Arr
)) {
3564 tvCastHelper
= (TCA
)tvCastToArrayInPlace
;
3565 } else if (type
.subtypeOf(Type::Str
)) {
3566 tvCastHelper
= (TCA
)tvCastToStringInPlace
;
3567 } else if (type
.subtypeOf(Type::Obj
)) {
3568 tvCastHelper
= (TCA
)tvCastToObjectInPlace
;
3572 cgCallHelper(m_as
, tvCastHelper
, nullptr,
3573 SyncOptions::kSyncPoint
, args
, DestType::None
);
3576 void CodeGenerator::cgCoerceStk(IRInstruction
*inst
) {
3577 Type type
= inst
->typeParam();
3578 SSATmp
* sp
= inst
->src(0);
3579 uint32_t offset
= inst
->extra
<CoerceStk
>()->offset
;
3580 Block
* exit
= inst
->taken();
3581 PhysReg spReg
= m_regs
[sp
].reg();
3583 ArgGroup
args(m_regs
);
3584 args
.addr(spReg
, cellsToBytes(offset
));
3587 if (type
.subtypeOf(Type::Bool
)) {
3588 tvCoerceHelper
= (TCA
)tvCoerceParamToBooleanInPlace
;
3589 } else if (type
.subtypeOf(Type::Int
)) {
3590 // if casting to integer, pass 10 as the base for the conversion
3592 tvCoerceHelper
= (TCA
)tvCoerceParamToInt64InPlace
;
3593 } else if (type
.subtypeOf(Type::Dbl
)) {
3594 tvCoerceHelper
= (TCA
)tvCoerceParamToDoubleInPlace
;
3595 } else if (type
.subtypeOf(Type::Arr
)) {
3596 tvCoerceHelper
= (TCA
)tvCoerceParamToArrayInPlace
;
3597 } else if (type
.subtypeOf(Type::Str
)) {
3598 tvCoerceHelper
= (TCA
)tvCoerceParamToStringInPlace
;
3599 } else if (type
.subtypeOf(Type::Obj
)) {
3600 tvCoerceHelper
= (TCA
)tvCoerceParamToObjectInPlace
;
3605 auto tmpReg
= PhysReg(m_rScratch
);
3606 cgCallHelper(m_as
, tvCoerceHelper
, tmpReg
, SyncOptions::kSyncPoint
, args
);
3607 m_as
.testb(1, rbyte(tmpReg
));
3608 emitFwdJcc(m_as
, CC_E
, exit
);
3611 void CodeGenerator::cgCallBuiltin(IRInstruction
* inst
) {
3612 SSATmp
* f
= inst
->src(0);
3613 auto args
= inst
->srcs().subpiece(2);
3614 int32_t numArgs
= args
.size();
3615 SSATmp
* dst
= inst
->dst();
3616 auto dstReg
= m_regs
[dst
].reg(0);
3617 auto dstType
= m_regs
[dst
].reg(1);
3618 Type returnType
= inst
->typeParam();
3620 const Func
* func
= f
->getValFunc();
3621 DataType funcReturnType
= func
->returnType();
3622 int returnOffset
= HHIR_MISOFF(tvBuiltinReturn
);
3624 if (TranslatorX64::eagerRecord(func
)) {
3625 const uchar
* pc
= curUnit()->entry() + m_curInst
->marker().bcOff
;
3626 // we have spilled all args to stack, so spDiff is 0
3627 m_tx64
->emitEagerSyncPoint(m_as
, pc
, 0);
3629 // RSP points to the MInstrState we need to use.
3630 // workaround the fact that rsp moves when we spill registers around call
3631 PhysReg misReg
= m_rScratch
;
3632 emitMovRegReg(m_as
, reg::rsp
, misReg
);
3634 ArgGroup
callArgs(m_regs
);
3635 if (isCppByRef(funcReturnType
)) {
3636 // first arg is pointer to storage for that return value
3637 if (isSmartPtrRef(funcReturnType
)) {
3638 returnOffset
+= TVOFF(m_data
);
3640 // misReg is pointing to an MInstrState struct on the C stack. Pass
3641 // the address of tvBuiltinReturn to the native function as the location
3642 // it can construct the return Array, String, Object, or Variant.
3643 callArgs
.addr(misReg
, returnOffset
); // &misReg[returnOffset]
3646 // non-pointer args are plain values passed by value. String, Array,
3647 // Object, and Variant are passed by const&, ie a pointer to stack memory
3648 // holding the value, so expect PtrToT types for these.
3649 // Pointers to smartptr types (String, Array, Object) need adjusting to
3650 // point to &ptr->m_data.
3651 for (int i
= 0; i
< numArgs
; i
++) {
3652 const Func::ParamInfo
& pi
= func
->params()[i
];
3653 if (TVOFF(m_data
) && isSmartPtrRef(pi
.builtinType())) {
3654 assert(args
[i
]->type().isPtr() && m_regs
[args
[i
]].reg() != InvalidReg
);
3655 callArgs
.addr(m_regs
[args
[i
]].reg(), TVOFF(m_data
));
3657 callArgs
.ssa(args
[i
]);
3661 // if the return value is returned by reference, we don't need the
3662 // return value from this call since we know where the value is.
3663 cgCallHelper(m_as
, Transl::CppCall((TCA
)func
->nativeFuncPtr()),
3664 isCppByRef(funcReturnType
) ? InvalidReg
: dstReg
,
3665 SyncOptions::kSyncPoint
, callArgs
);
3667 // load return value from builtin
3668 // for primitive return types (int, bool), the return value
3669 // is already in dstReg (the builtin call returns in rax). For return
3670 // by reference (String, Object, Array, Variant), the builtin writes the
3671 // return value into MInstrState::tvBuiltinReturn TV, from where it
3672 // has to be tested and copied.
3673 if (dstReg
== InvalidReg
|| returnType
.isSimpleType()) {
3676 // after the call, RSP is back pointing to MInstrState and rSratch
3677 // has been clobberred.
3680 if (returnType
.isReferenceType()) {
3681 assert(isCppByRef(funcReturnType
) && isSmartPtrRef(funcReturnType
));
3682 // return type is String, Array, or Object; fold nullptr to KindOfNull
3683 m_as
. loadq (misReg
[returnOffset
], dstReg
);
3684 emitLoadImm(m_as
, returnType
.toDataType(), dstType
);
3685 emitLoadImm(m_as
, KindOfNull
, m_rScratch
);
3686 m_as
. testq (dstReg
, dstReg
);
3687 m_as
. cmov_reg64_reg64 (CC_Z
, m_rScratch
, dstType
);
3690 if (returnType
.subtypeOf(Type::Cell
)
3691 || returnType
.subtypeOf(Type::BoxedCell
)) {
3692 // return type is Variant; fold KindOfUninit to KindOfNull
3693 assert(isCppByRef(funcReturnType
) && !isSmartPtrRef(funcReturnType
));
3694 assert(misReg
!= dstType
);
3695 emitLoadTVType(m_as
, misReg
[returnOffset
+ TVOFF(m_type
)], dstType
);
3696 m_as
. loadq (misReg
[returnOffset
+ TVOFF(m_data
)], dstReg
);
3697 emitLoadImm(m_as
, KindOfNull
, m_rScratch
);
3698 static_assert(KindOfUninit
== 0, "KindOfUninit must be 0 for test");
3699 m_as
. testb (rbyte(dstType
), rbyte(dstType
));
3700 m_as
. cmov_reg64_reg64 (CC_Z
, m_rScratch
, dstType
);
3706 void CodeGenerator::cgSpillStack(IRInstruction
* inst
) {
3707 SSATmp
* dst
= inst
->dst();
3708 SSATmp
* sp
= inst
->src(0);
3709 auto const spDeficit
= inst
->src(1)->getValInt();
3710 auto const spillVals
= inst
->srcs().subpiece(2);
3711 auto const numSpillSrcs
= spillVals
.size();
3712 auto const dstReg
= m_regs
[dst
].reg();
3713 auto const spReg
= m_regs
[sp
].reg();
3714 auto const spillCells
= spillValueCells(inst
);
3716 int64_t adjustment
= (spDeficit
- spillCells
) * sizeof(Cell
);
3717 for (uint32_t i
= 0; i
< numSpillSrcs
; ++i
) {
3718 const int64_t offset
= i
* sizeof(Cell
) + adjustment
;
3719 if (spillVals
[i
]->type() == Type::None
) {
3720 // The simplifier detected that we're storing the same value
3721 // already in there.
3725 auto* val
= spillVals
[i
];
3726 auto* inst
= val
->inst();
3727 while (inst
->isPassthrough()) {
3728 inst
= inst
->getPassthroughValue()->inst();
3730 // If our value came from a LdStack on the same sp and offset,
3731 // we don't need to spill it.
3732 if (inst
->op() == LdStack
&& inst
->src(0) == sp
&&
3733 inst
->extra
<LdStack
>()->offset
* sizeof(Cell
) == offset
) {
3734 FTRACE(6, "{}: Not spilling spill value {} from {}\n",
3735 __func__
, i
, inst
->toString());
3737 cgStore(spReg
, offset
, val
);
3741 emitAdjustSp(spReg
, dstReg
, adjustment
);
3744 void CodeGenerator::emitAdjustSp(PhysReg spReg
,
3746 int64_t adjustment
/* bytes */) {
3747 if (adjustment
!= 0) {
3748 if (dstReg
!= spReg
) {
3749 m_as
. lea (spReg
[adjustment
], dstReg
);
3751 m_as
. addq (adjustment
, dstReg
);
3754 emitMovRegReg(m_as
, spReg
, dstReg
);
3758 void CodeGenerator::cgNativeImpl(IRInstruction
* inst
) {
3759 SSATmp
* func
= inst
->src(0);
3760 SSATmp
* fp
= inst
->src(1);
3762 assert(func
->isConst());
3763 assert(func
->type() == Type::Func
);
3764 const Func
* fn
= func
->getValFunc();
3766 BuiltinFunction builtinFuncPtr
= func
->getValFunc()->builtinFuncPtr();
3767 emitMovRegReg(m_as
, m_regs
[fp
].reg(), argNumToRegName
[0]);
3768 if (TranslatorX64::eagerRecord(fn
)) {
3769 m_tx64
->emitEagerSyncPoint(m_as
, fn
->getEntry(), 0);
3771 m_as
.call((TCA
)builtinFuncPtr
);
3772 recordSyncPoint(m_as
);
3775 void CodeGenerator::cgLdThis(IRInstruction
* inst
) {
3776 SSATmp
* dst
= inst
->dst();
3777 SSATmp
* src
= inst
->src(0);
3778 Block
* label
= inst
->taken();
3779 // mov dst, [fp + 0x20]
3780 auto dstReg
= m_regs
[dst
].reg();
3782 // the destination of LdThis could be dead but the instruction
3783 // itself still useful because of the checks that it does (if it has
3784 // a label). So we need to make sure there is a dstReg for this
3786 if (dstReg
!= InvalidReg
) {
3787 // instruction's result is not dead
3788 m_as
.loadq(m_regs
[src
].reg()[AROFF(m_this
)], dstReg
);
3790 if (label
== NULL
) return; // no need to perform its checks
3791 if (dstReg
!= InvalidReg
) {
3793 m_as
.testb(1, rbyte(dstReg
));
3795 m_as
.testb(1, m_regs
[src
].reg()[AROFF(m_this
)]);
3798 emitFwdJcc(CC_NZ
, label
);
3801 static void emitLdClsCctx(CodeGenerator::Asm
& a
,
3804 emitMovRegReg(a
, srcReg
, dstReg
);
3808 void CodeGenerator::cgLdClsCtx(IRInstruction
* inst
) {
3809 PhysReg srcReg
= m_regs
[inst
->src(0)].reg();
3810 PhysReg dstReg
= m_regs
[inst
->dst()].reg();
3811 // Context could be either a this object or a class ptr
3812 m_as
. testb(1, rbyte(srcReg
));
3814 [&] { emitLdClsCctx(m_as
, srcReg
, dstReg
); }, // ctx is a class
3815 [&] { emitLdObjClass(m_as
, srcReg
, dstReg
); } // ctx is this ptr
3819 void CodeGenerator::cgLdClsCctx(IRInstruction
* inst
) {
3820 PhysReg srcReg
= m_regs
[inst
->src(0)].reg();
3821 PhysReg dstReg
= m_regs
[inst
->dst()].reg();
3822 emitLdClsCctx(m_as
, srcReg
, dstReg
);
3825 void CodeGenerator::cgLdCtx(IRInstruction
* inst
) {
3826 PhysReg dstReg
= m_regs
[inst
->dst()].reg();
3827 PhysReg srcReg
= m_regs
[inst
->src(0)].reg();
3828 if (dstReg
!= InvalidReg
) {
3829 m_as
.loadq(srcReg
[AROFF(m_this
)], dstReg
);
3833 void CodeGenerator::cgLdCctx(IRInstruction
* inst
) {
3834 return cgLdCtx(inst
);
3837 void CodeGenerator::cgLdConst(IRInstruction
* inst
) {
3838 auto const dstReg
= m_regs
[inst
->dst()].reg();
3839 auto const val
= inst
->extra
<LdConst
>()->as
<uintptr_t>();
3840 if (dstReg
== InvalidReg
) return;
3841 emitLoadImm(m_as
, val
, dstReg
);
3844 void CodeGenerator::cgLdARFuncPtr(IRInstruction
* inst
) {
3845 SSATmp
* dst
= inst
->dst();
3846 SSATmp
* baseAddr
= inst
->src(0);
3847 SSATmp
* offset
= inst
->src(1);
3849 auto dstReg
= m_regs
[dst
].reg();
3850 auto baseReg
= m_regs
[baseAddr
].reg();
3852 assert(offset
->isConst());
3854 m_as
.load_reg64_disp_reg64(baseReg
,
3855 offset
->getValInt() + AROFF(m_func
),
3859 static int getNativeTypeSize(Type type
) {
3860 if (type
.subtypeOf(Type::Int
| Type::Func
)) return sz::qword
;
3861 if (type
.subtypeOf(Type::Bool
)) return sz::byte
;
3865 void CodeGenerator::cgLdRaw(IRInstruction
* inst
) {
3866 SSATmp
* dest
= inst
->dst();
3867 SSATmp
* addr
= inst
->src(0);
3868 SSATmp
* offset
= inst
->src(1);
3870 assert(!(dest
->isConst()));
3872 Reg64 addrReg
= m_regs
[addr
].reg();
3873 PhysReg destReg
= m_regs
[dest
].reg();
3875 if (addr
->isConst()) {
3876 addrReg
= m_rScratch
;
3877 emitLoadImm(m_as
, addr
->getValRawInt(), addrReg
);
3880 if (offset
->isConst()) {
3881 assert(offset
->type() == Type::Int
);
3882 int64_t kind
= offset
->getValInt();
3883 RawMemSlot
& slot
= RawMemSlot::Get(RawMemSlot::Kind(kind
));
3884 int ldSize
= slot
.size();
3885 int64_t off
= slot
.offset();
3886 if (ldSize
== sz::qword
) {
3887 m_as
.loadq (addrReg
[off
], destReg
);
3888 } else if (ldSize
== sz::dword
) {
3889 m_as
.loadl (addrReg
[off
], r32(destReg
));
3891 assert(ldSize
== sz::byte
);
3892 m_as
.loadzbl (addrReg
[off
], r32(destReg
));
3895 int ldSize
= getNativeTypeSize(dest
->type());
3896 Reg64 offsetReg
= r64(m_regs
[offset
].reg());
3897 if (ldSize
== sz::qword
) {
3898 m_as
.loadq (addrReg
[offsetReg
], destReg
);
3900 // Not yet supported by our assembler
3901 assert(ldSize
== sz::byte
);
3907 void CodeGenerator::cgStRaw(IRInstruction
* inst
) {
3908 auto baseReg
= m_regs
[inst
->src(0)].reg();
3909 int64_t kind
= inst
->src(1)->getValInt();
3910 SSATmp
* value
= inst
->src(2);
3912 RawMemSlot
& slot
= RawMemSlot::Get(RawMemSlot::Kind(kind
));
3913 assert(value
->type().equals(slot
.type()));
3914 int stSize
= slot
.size();
3915 int64_t off
= slot
.offset();
3916 auto dest
= baseReg
[off
];
3918 if (value
->isConst()) {
3919 if (stSize
== sz::qword
) {
3920 m_as
.storeq(value
->getValRawInt(), dest
);
3921 } else if (stSize
== sz::dword
) {
3922 m_as
.storel(value
->getValRawInt(), dest
);
3924 assert(stSize
== sz::byte
);
3925 m_as
.storeb(value
->getValBool(), dest
);
3928 if (stSize
== sz::qword
) {
3929 m_as
.storeq(r64(m_regs
[value
].reg()), dest
);
3930 } else if (stSize
== sz::dword
) {
3931 m_as
.storel(r32(m_regs
[value
].reg()), dest
);
3933 assert(stSize
== sz::byte
);
3934 m_as
.storeb(rbyte(m_regs
[value
].reg()), dest
);
3939 void CodeGenerator::cgLdStaticLocCached(IRInstruction
* inst
) {
3940 auto ch
= inst
->src(0)->getValRawInt();
3941 auto outReg
= m_regs
[inst
->dst()].reg();
3943 m_as
.loadq (rVmTl
[ch
], outReg
);
3944 m_as
.testq (outReg
, outReg
);
3945 emitFwdJcc(m_as
, CC_Z
, inst
->taken());
3948 // If label is set and type is not Gen, this method generates a check
3949 // that bails to the label if the loaded typed value doesn't match type.
3950 void CodeGenerator::cgLoadTypedValue(PhysReg base
,
3952 IRInstruction
* inst
) {
3953 Block
* label
= inst
->taken();
3954 Type type
= inst
->typeParam();
3955 SSATmp
* dst
= inst
->dst();
3957 assert(type
== dst
->type());
3958 assert(type
.needsReg());
3959 auto valueDstReg
= m_regs
[dst
].reg(0);
3960 auto typeDstReg
= m_regs
[dst
].reg(1);
3962 if (valueDstReg
.isXMM()) {
3963 // Whole typed value is stored in single XMM reg valueDstReg
3964 assert(RuntimeOption::EvalHHIRAllocXMMRegs
);
3965 assert(typeDstReg
== InvalidReg
);
3966 m_as
.movdqa(base
[off
+ TVOFF(m_data
)], valueDstReg
);
3970 if (valueDstReg
== InvalidReg
&& typeDstReg
== InvalidReg
&&
3971 (label
== nullptr || type
== Type::Gen
)) {
3975 bool useScratchReg
= (base
== typeDstReg
&& valueDstReg
!= InvalidReg
);
3976 if (useScratchReg
) {
3977 // Save base to m_rScratch, because base will be overwritten.
3978 m_as
.mov_reg64_reg64(base
, m_rScratch
);
3981 // Load type if it's not dead
3982 if (typeDstReg
!= InvalidReg
) {
3983 emitLoadTVType(m_as
, base
[off
+ TVOFF(m_type
)], typeDstReg
);
3985 emitTypeCheck(inst
->typeParam(), typeDstReg
,
3986 valueDstReg
, inst
->taken());
3989 emitTypeCheck(inst
->typeParam(),
3990 base
[off
+ TVOFF(m_type
)],
3991 base
[off
+ TVOFF(m_data
)],
3995 // Load value if it's not dead
3996 if (valueDstReg
== InvalidReg
) return;
3998 if (useScratchReg
) {
3999 m_as
.loadq(m_rScratch
[off
+ TVOFF(m_data
)], valueDstReg
);
4001 m_as
.loadq(base
[off
+ TVOFF(m_data
)], valueDstReg
);
4005 void CodeGenerator::cgStoreTypedValue(PhysReg base
,
4008 assert(src
->type().needsReg());
4009 auto srcReg0
= m_regs
[src
].reg(0);
4010 auto srcReg1
= m_regs
[src
].reg(1);
4011 if (srcReg0
.isXMM()) {
4012 // Whole typed value is stored in single XMM reg srcReg0
4013 assert(RuntimeOption::EvalHHIRAllocXMMRegs
);
4014 assert(srcReg1
== InvalidReg
);
4015 m_as
.movdqa(srcReg0
, base
[off
+ TVOFF(m_data
)]);
4018 m_as
.storeq(srcReg0
, base
[off
+ TVOFF(m_data
)]);
4019 emitStoreTVType(m_as
, srcReg1
, base
[off
+ TVOFF(m_type
)]);
4022 void CodeGenerator::cgStore(PhysReg base
,
4025 bool genStoreType
) {
4026 Type type
= src
->type();
4027 if (type
.needsReg()) {
4028 cgStoreTypedValue(base
, off
, src
);
4033 emitStoreTVType(m_as
, type
.toDataType(), base
[off
+ TVOFF(m_type
)]);
4035 if (type
.isNull()) {
4036 // no need to store a value for null or uninit
4039 if (src
->isConst()) {
4041 if (type
.subtypeOf(Type::Bool
| Type::Int
| Type::Dbl
|
4042 Type::Arr
| Type::StaticStr
| Type::Cls
)) {
4043 val
= src
->getValBits();
4047 m_as
.storeq(val
, base
[off
+ TVOFF(m_data
)]);
4049 zeroExtendIfBool(m_as
, src
, m_regs
[src
]);
4050 emitStoreReg(m_as
, m_regs
[src
].reg(), base
[off
+ TVOFF(m_data
)]);
4054 void CodeGenerator::cgLoad(PhysReg base
,
4056 IRInstruction
* inst
) {
4057 Type type
= inst
->typeParam();
4058 if (type
.needsReg()) {
4059 return cgLoadTypedValue(base
, off
, inst
);
4061 Block
* label
= inst
->taken();
4062 if (label
!= NULL
) {
4063 emitTypeCheck(inst
->typeParam(),
4064 base
[off
+ TVOFF(m_type
)],
4065 base
[off
+ TVOFF(m_data
)],
4068 if (type
.isNull()) return; // these are constants
4069 auto dstReg
= m_regs
[inst
->dst()].reg();
4070 // if dstReg == InvalidReg then the value of this load is dead
4071 if (dstReg
== InvalidReg
) return;
4073 if (type
== Type::Bool
) {
4074 m_as
.load_reg64_disp_reg32(base
, off
+ TVOFF(m_data
), dstReg
);
4076 emitLoadReg(m_as
, base
[off
+ TVOFF(m_data
)], dstReg
);
4080 void CodeGenerator::cgLdProp(IRInstruction
* inst
) {
4081 cgLoad(m_regs
[inst
->src(0)].reg(), inst
->src(1)->getValInt(), inst
);
4084 void CodeGenerator::cgLdMem(IRInstruction
* inst
) {
4085 cgLoad(m_regs
[inst
->src(0)].reg(), inst
->src(1)->getValInt(), inst
);
4088 void CodeGenerator::cgLdRef(IRInstruction
* inst
) {
4089 cgLoad(m_regs
[inst
->src(0)].reg(), RefData::tvOffset(), inst
);
4092 void CodeGenerator::recordSyncPoint(Asm
& as
,
4093 SyncOptions sync
/* = kSyncPoint */) {
4094 assert(m_curInst
->marker().valid());
4096 Offset stackOff
= m_curInst
->marker().spOff
;
4098 case SyncOptions::kSyncPointAdjustOne
:
4101 case SyncOptions::kSyncPoint
:
4103 case SyncOptions::kNoSyncPoint
:
4107 Offset pcOff
= m_curInst
->marker().bcOff
- m_curInst
->marker().func
->base();
4109 FTRACE(5, "IR recordSyncPoint: {} {} {}\n", as
.code
.frontier
, pcOff
,
4111 m_tx64
->recordSyncPoint(as
, pcOff
, stackOff
);
4114 void CodeGenerator::cgLdAddr(IRInstruction
* inst
) {
4115 auto base
= m_regs
[inst
->src(0)].reg();
4116 int64_t offset
= inst
->src(1)->getValInt();
4117 m_as
.lea (base
[offset
], m_regs
[inst
->dst()].reg());
4120 void CodeGenerator::cgLdLoc(IRInstruction
* inst
) {
4121 cgLoad(m_regs
[inst
->src(0)].reg(),
4122 localOffset(inst
->extra
<LdLoc
>()->locId
),
4126 void CodeGenerator::cgLdLocAddr(IRInstruction
* inst
) {
4127 auto const fpReg
= m_regs
[inst
->src(0)].reg();
4128 auto const offset
= localOffset(inst
->extra
<LdLocAddr
>()->locId
);
4129 if (m_regs
[inst
->dst()].hasReg()) {
4130 m_as
.lea(fpReg
[offset
], m_regs
[inst
->dst()].reg());
4134 void CodeGenerator::cgLdStackAddr(IRInstruction
* inst
) {
4135 auto const base
= m_regs
[inst
->src(0)].reg();
4136 auto const offset
= cellsToBytes(inst
->extra
<LdStackAddr
>()->offset
);
4137 m_as
.lea (base
[offset
], m_regs
[inst
->dst()].reg());
4140 void CodeGenerator::cgLdStack(IRInstruction
* inst
) {
4141 assert(inst
->taken() == nullptr);
4142 cgLoad(m_regs
[inst
->src(0)].reg(),
4143 cellsToBytes(inst
->extra
<LdStack
>()->offset
),
4147 void CodeGenerator::cgGuardStk(IRInstruction
* inst
) {
4148 auto const rSP
= m_regs
[inst
->src(0)].reg();
4149 auto const baseOff
= cellsToBytes(inst
->extra
<GuardStk
>()->offset
);
4150 emitTypeGuard(inst
->typeParam(),
4151 rSP
[baseOff
+ TVOFF(m_type
)],
4152 rSP
[baseOff
+ TVOFF(m_data
)]);
4155 void CodeGenerator::cgCheckStk(IRInstruction
* inst
) {
4156 auto const rbase
= m_regs
[inst
->src(0)].reg();
4157 auto const baseOff
= cellsToBytes(inst
->extra
<CheckStk
>()->offset
);
4158 emitTypeCheck(inst
->typeParam(), rbase
[baseOff
+ TVOFF(m_type
)],
4159 rbase
[baseOff
+ TVOFF(m_data
)], inst
->taken());
4162 void CodeGenerator::cgGuardLoc(IRInstruction
* inst
) {
4163 auto const rFP
= m_regs
[inst
->src(0)].reg();
4164 auto const baseOff
= localOffset(inst
->extra
<GuardLoc
>()->locId
);
4165 emitTypeGuard(inst
->typeParam(),
4166 rFP
[baseOff
+ TVOFF(m_type
)],
4167 rFP
[baseOff
+ TVOFF(m_data
)]);
4170 void CodeGenerator::cgCheckLoc(IRInstruction
* inst
) {
4171 auto const rbase
= m_regs
[inst
->src(0)].reg();
4172 auto const baseOff
= localOffset(inst
->extra
<CheckLoc
>()->locId
);
4173 emitTypeCheck(inst
->typeParam(), rbase
[baseOff
+ TVOFF(m_type
)],
4174 rbase
[baseOff
+ TVOFF(m_data
)], inst
->taken());
4178 void CodeGenerator::emitSideExitGuard(Type type
,
4182 emitTypeTest(type
, typeSrc
, dataSrc
,
4183 [&](ConditionCode cc
) {
4184 auto const sk
= SrcKey(curFunc(), taken
);
4185 m_tx64
->emitBindJcc(m_as
, ccNegate(cc
), sk
, REQ_BIND_SIDE_EXIT
);
4189 void CodeGenerator::cgSideExitGuardLoc(IRInstruction
* inst
) {
4190 auto const fp
= m_regs
[inst
->src(0)].reg();
4191 auto const extra
= inst
->extra
<SideExitGuardLoc
>();
4192 emitSideExitGuard(inst
->typeParam(),
4193 fp
[localOffset(extra
->checkedSlot
) + TVOFF(m_type
)],
4194 fp
[localOffset(extra
->checkedSlot
) + TVOFF(m_data
)],
4198 void CodeGenerator::cgSideExitGuardStk(IRInstruction
* inst
) {
4199 auto const sp
= m_regs
[inst
->src(0)].reg();
4200 auto const extra
= inst
->extra
<SideExitGuardStk
>();
4201 emitSideExitGuard(inst
->typeParam(),
4202 sp
[cellsToBytes(extra
->checkedSlot
) + TVOFF(m_type
)],
4203 sp
[cellsToBytes(extra
->checkedSlot
) + TVOFF(m_data
)],
4207 void CodeGenerator::cgDefMIStateBase(IRInstruction
* inst
) {
4208 assert(inst
->dst()->type() == Type::PtrToCell
);
4209 assert(m_regs
[inst
->dst()].reg() == rsp
);
4212 void CodeGenerator::cgCheckType(IRInstruction
* inst
) {
4213 auto const src
= inst
->src(0);
4214 auto const t
= inst
->typeParam();
4215 auto const rData
= m_regs
[src
].reg(0);
4216 auto const rType
= m_regs
[src
].reg(1);
4218 auto doJcc
= [&](ConditionCode cc
) {
4219 emitFwdJcc(ccNegate(cc
), inst
->taken());
4222 if (t
.equals(Type::Nullptr
)) {
4223 if (!src
->type().equals(Type::Nullptr
| Type::CountedStr
)) {
4224 CG_PUNT(CheckType
-Nullptr
-UnsupportedType
);
4226 m_as
.testq (rData
, rData
);
4229 emitTypeTest(inst
->typeParam(), rType
, rData
, doJcc
);
4232 auto const dstReg
= m_regs
[inst
->dst()].reg();
4233 if (dstReg
!= InvalidReg
) {
4234 emitMovRegReg(m_as
, rData
, dstReg
);
4238 void CodeGenerator::cgCheckTypeMem(IRInstruction
* inst
) {
4239 auto const reg
= m_regs
[inst
->src(0)].reg();
4240 emitTypeCheck(inst
->typeParam(), reg
[TVOFF(m_type
)],
4241 reg
[TVOFF(m_data
)], inst
->taken());
4244 void CodeGenerator::cgCheckDefinedClsEq(IRInstruction
* inst
) {
4245 auto const clsName
= inst
->extra
<CheckDefinedClsEq
>()->clsName
;
4246 auto const cls
= inst
->extra
<CheckDefinedClsEq
>()->cls
;
4247 auto const ch
= TargetCache::allocKnownClass(clsName
);
4248 m_as
.cmpq(cls
, rVmTl
[ch
]);
4249 emitFwdJcc(CC_NZ
, inst
->taken());
4252 void CodeGenerator::cgGuardRefs(IRInstruction
* inst
) {
4253 assert(inst
->numSrcs() == 5);
4255 SSATmp
* funcPtrTmp
= inst
->src(0);
4256 SSATmp
* nParamsTmp
= inst
->src(1);
4257 SSATmp
* firstBitNumTmp
= inst
->src(2);
4258 SSATmp
* mask64Tmp
= inst
->src(3);
4259 SSATmp
* vals64Tmp
= inst
->src(4);
4261 // Get values in place
4262 assert(funcPtrTmp
->type() == Type::Func
);
4263 auto funcPtrReg
= m_regs
[funcPtrTmp
].reg();
4264 assert(funcPtrReg
!= InvalidReg
);
4266 assert(nParamsTmp
->type() == Type::Int
);
4267 auto nParamsReg
= m_regs
[nParamsTmp
].reg();
4268 assert(nParamsReg
!= InvalidReg
|| nParamsTmp
->isConst());
4270 assert(firstBitNumTmp
->isConst() && firstBitNumTmp
->type() == Type::Int
);
4271 uint32_t firstBitNum
= (uint32_t)(firstBitNumTmp
->getValInt());
4273 assert(mask64Tmp
->type() == Type::Int
);
4274 assert(mask64Tmp
->isConst());
4275 auto mask64Reg
= m_regs
[mask64Tmp
].reg();
4276 assert(mask64Reg
!= InvalidReg
|| mask64Tmp
->inst()->op() != LdConst
);
4277 uint64_t mask64
= mask64Tmp
->getValInt();
4280 assert(vals64Tmp
->type() == Type::Int
);
4281 assert(vals64Tmp
->isConst());
4282 auto vals64Reg
= m_regs
[vals64Tmp
].reg();
4283 assert(vals64Reg
!= InvalidReg
|| vals64Tmp
->inst()->op() != LdConst
);
4284 uint64_t vals64
= vals64Tmp
->getValInt();
4285 assert((vals64
& mask64
) == vals64
);
4287 auto const destSK
= SrcKey(curFunc(), m_curTrace
->bcOff());
4288 auto const destSR
= m_tx64
->getSrcRec(destSK
);
4290 auto thenBody
= [&] {
4291 auto bitsOff
= sizeof(uint64_t) * (firstBitNum
/ 64);
4293 auto bitsPtrReg
= m_rScratch
;
4295 if (firstBitNum
== 0) {
4296 bitsOff
= Func::refBitValOff();
4297 bitsPtrReg
= funcPtrReg
;
4299 m_as
.loadq(funcPtrReg
[Func::sharedOff()], bitsPtrReg
);
4300 bitsOff
-= sizeof(uint64_t);
4303 if (vals64
== 0 || (mask64
& (mask64
- 1)) == 0) {
4304 // If vals64 is zero, or we're testing a single
4305 // bit, we can get away with a single test,
4306 // rather than mask-and-compare
4307 if (mask64Reg
!= InvalidReg
) {
4308 m_as
.testq (mask64Reg
, bitsPtrReg
[bitsOff
]);
4311 m_as
.testb((int8_t)mask64
, bitsPtrReg
[bitsOff
]);
4313 m_as
.testl((int32_t)mask64
, bitsPtrReg
[bitsOff
]);
4316 if (vals64
) cond
= CC_E
;
4318 auto bitsValReg
= m_rScratch
;
4319 m_as
. loadq (bitsPtrReg
[bitsOff
], bitsValReg
);
4320 if (debug
) bitsPtrReg
= InvalidReg
;
4322 // bitsValReg <- bitsValReg & mask64
4323 if (mask64Reg
!= InvalidReg
) {
4324 m_as
. andq (mask64Reg
, bitsValReg
);
4325 } else if (mask64
< 256) {
4326 m_as
. andb ((int8_t)mask64
, rbyte(bitsValReg
));
4328 m_as
. andl ((int32_t)mask64
, r32(bitsValReg
));
4331 // If bitsValReg != vals64, then goto Exit
4332 if (vals64Reg
!= InvalidReg
) {
4333 m_as
. cmpq (vals64Reg
, bitsValReg
);
4334 } else if (mask64
< 256) {
4335 assert(vals64
< 256);
4336 m_as
. cmpb ((int8_t)vals64
, rbyte(bitsValReg
));
4338 m_as
. cmpl ((int32_t)vals64
, r32(bitsValReg
));
4341 m_tx64
->emitFallbackCondJmp(m_as
, *destSR
, cond
);
4344 if (firstBitNum
== 0) {
4345 assert(nParamsReg
== InvalidReg
);
4346 // This is the first 64 bits. No need to check
4350 assert(nParamsReg
!= InvalidReg
);
4351 // Check number of args...
4352 m_as
. cmpq (firstBitNum
, nParamsReg
);
4354 if (vals64
!= 0 && vals64
!= mask64
) {
4355 // If we're beyond nParams, then either all params
4356 // are refs, or all params are non-refs, so if vals64
4357 // isn't 0 and isnt mask64, there's no possibility of
4359 m_tx64
->emitFallbackCondJmp(m_as
, *destSR
, CC_LE
);
4362 ifThenElse(CC_NLE
, thenBody
, /* else */ [&] {
4363 // If not special builtin...
4364 m_as
.testl(AttrVariadicByRef
, funcPtrReg
[Func::attrsOff()]);
4365 m_tx64
->emitFallbackCondJmp(m_as
, *destSR
, vals64
? CC_Z
: CC_NZ
);
4371 void CodeGenerator::cgLdPropAddr(IRInstruction
* inst
) {
4372 SSATmp
* dst
= inst
->dst();
4373 SSATmp
* obj
= inst
->src(0);
4374 SSATmp
* prop
= inst
->src(1);
4376 assert(prop
->isConst() && prop
->type() == Type::Int
);
4378 auto dstReg
= m_regs
[dst
].reg();
4379 auto objReg
= m_regs
[obj
].reg();
4381 assert(objReg
!= InvalidReg
);
4382 assert(dstReg
!= InvalidReg
);
4384 int64_t offset
= prop
->getValInt();
4385 m_as
.lea_reg64_disp_reg64(objReg
, offset
, dstReg
);
4388 void CodeGenerator::cgLdClsMethod(IRInstruction
* inst
) {
4389 SSATmp
* dst
= inst
->dst();
4390 SSATmp
* cls
= inst
->src(0);
4391 SSATmp
* mSlot
= inst
->src(1);
4393 assert(cls
->type() == Type::Cls
);
4394 assert(mSlot
->isConst() && mSlot
->type() == Type::Int
);
4395 uint64_t mSlotInt64
= mSlot
->getValRawInt();
4396 // We're going to multiply mSlotVal by sizeof(Func*) and use
4397 // it as a 32-bit offset (methOff) below.
4398 if (mSlotInt64
> (std::numeric_limits
<uint32_t>::max() / sizeof(Func
*))) {
4399 CG_PUNT(cgLdClsMethod_large_offset
);
4401 int32_t mSlotVal
= (uint32_t) mSlotInt64
;
4403 Reg64 dstReg
= m_regs
[dst
].reg();
4404 assert(dstReg
!= InvalidReg
);
4406 Reg64 clsReg
= m_regs
[cls
].reg();
4407 if (clsReg
== InvalidReg
) {
4408 CG_PUNT(LdClsMethod
);
4411 Offset vecOff
= Class::getMethodsOffset() + Class::MethodMap::vecOff();
4412 int32_t methOff
= mSlotVal
* sizeof(Func
*);
4413 m_as
.loadq(clsReg
[vecOff
], dstReg
);
4414 m_as
.loadq(dstReg
[methOff
], dstReg
);
4417 void CodeGenerator::cgLdClsMethodCache(IRInstruction
* inst
) {
4418 SSATmp
* dst
= inst
->dst();
4419 SSATmp
* className
= inst
->src(0);
4420 SSATmp
* methodName
= inst
->src(1);
4421 SSATmp
* baseClass
= inst
->src(2);
4422 SSATmp
* fp
= inst
->src(3);
4423 SSATmp
* sp
= inst
->src(4);
4424 Block
* label
= inst
->taken();
4426 // Stats::emitInc(a, Stats::TgtCache_StaticMethodHit);
4427 const StringData
* cls
= className
->getValStr();
4428 const StringData
* method
= methodName
->getValStr();
4429 auto const ne
= baseClass
->getValNamedEntity();
4430 TargetCache::CacheHandle ch
=
4431 TargetCache::StaticMethodCache::alloc(cls
,
4433 getContextName(curClass()));
4434 auto funcDestReg
= m_regs
[dst
].reg(0);
4435 auto classDestReg
= m_regs
[dst
].reg(1);
4436 auto offsetof_func
= offsetof(TargetCache::StaticMethodCache
, m_func
);
4437 auto offsetof_cls
= offsetof(TargetCache::StaticMethodCache
, m_cls
);
4439 assert(funcDestReg
!= InvalidReg
&& classDestReg
!= InvalidReg
);
4440 // Attempt to retrieve the func* and class* from cache
4441 m_as
.loadq(rVmTl
[ch
+ offsetof_func
], funcDestReg
);
4442 m_as
.loadq(rVmTl
[ch
+ offsetof_cls
], classDestReg
);
4443 m_as
.testq(funcDestReg
, funcDestReg
);
4444 // May have retrieved a NULL from the cache
4445 // handle case where method is not entered in the cache
4446 unlikelyIfBlock(CC_E
, [&] (Asm
& a
) {
4447 if (false) { // typecheck
4448 UNUSED TypedValue
* fake_fp
= nullptr;
4449 UNUSED TypedValue
* fake_sp
= nullptr;
4450 const UNUSED Func
* f
= StaticMethodCache::lookupIR(ch
, ne
, cls
, method
,
4453 // can raise an error if class is undefined
4455 (TCA
)StaticMethodCache::lookupIR
,
4457 SyncOptions::kSyncPoint
,
4458 ArgGroup(m_regs
).imm(ch
) // Handle ch
4459 .immPtr(ne
) // NamedEntity* np.second
4460 .immPtr(cls
) // className
4461 .immPtr(method
) // methodName
4462 .reg(m_regs
[fp
].reg()) // frame pointer
4463 .reg(m_regs
[sp
].reg()) // stack pointer
4465 // recordInstrCall is done in cgCallHelper
4466 a
.testq(funcDestReg
, funcDestReg
);
4467 a
.loadq(rVmTl
[ch
+ offsetof_cls
], classDestReg
);
4468 // if StaticMethodCache::lookupIR() returned NULL, jmp to label
4469 emitFwdJcc(a
, CC_Z
, label
);
4474 * Helper to emit getting the value for ActRec's m_this/m_cls slot
4475 * from a This pointer depending on whether the callee method is
4478 void CodeGenerator::emitGetCtxFwdCallWithThis(PhysReg ctxReg
,
4479 bool staticCallee
) {
4481 // Load (this->m_cls | 0x1) into ctxReg.
4482 m_as
.loadq(ctxReg
[ObjectData::getVMClassOffset()], ctxReg
);
4483 m_as
.orq(1, ctxReg
);
4485 // Just incref $this.
4486 emitIncRef(m_as
, ctxReg
);
4491 * This method is similar to emitGetCtxFwdCallWithThis above, but
4492 * whether or not the callee is a static method is unknown at JIT
4493 * time, and that is determined dynamically by looking up into the
4494 * StaticMethodFCache.
4496 void CodeGenerator::emitGetCtxFwdCallWithThisDyn(PhysReg destCtxReg
,
4499 Label NonStaticCall
, End
;
4501 // thisReg is holding $this. Should we pass it to the callee?
4502 m_as
.cmpl(1, rVmTl
[ch
+ offsetof(StaticMethodFCache
, m_static
)]);
4503 m_as
.jcc8(CC_NE
, NonStaticCall
);
4504 // If calling a static method...
4506 // Load (this->m_cls | 0x1) into destCtxReg
4507 m_as
.loadq(thisReg
[ObjectData::getVMClassOffset()], destCtxReg
);
4508 m_as
.orq(1, destCtxReg
);
4511 // Else: calling non-static method
4513 asm_label(m_as
, NonStaticCall
);
4514 emitMovRegReg(m_as
, thisReg
, destCtxReg
);
4515 emitIncRef(m_as
, destCtxReg
);
4517 asm_label(m_as
, End
);
4520 void CodeGenerator::cgGetCtxFwdCall(IRInstruction
* inst
) {
4521 PhysReg destCtxReg
= m_regs
[inst
->dst()].reg(0);
4522 SSATmp
* srcCtxTmp
= inst
->src(0);
4523 const Func
* callee
= inst
->src(1)->getValFunc();
4524 bool withThis
= srcCtxTmp
->isA(Type::Obj
);
4526 // Eagerly move src into the dest reg
4527 emitMovRegReg(m_as
, m_regs
[srcCtxTmp
].reg(0), destCtxReg
);
4530 // If we don't know whether we have a This, we need to check dynamically
4532 m_as
.testb(1, rbyte(destCtxReg
));
4533 m_as
.jcc8(CC_NZ
, End
);
4536 // If we have a This pointer in destCtxReg, then select either This
4537 // or its Class based on whether callee is static or not
4538 emitGetCtxFwdCallWithThis(destCtxReg
, (callee
->attrs() & AttrStatic
));
4540 asm_label(m_as
, End
);
4543 void CodeGenerator::cgLdClsMethodFCache(IRInstruction
* inst
) {
4544 PhysReg funcDestReg
= m_regs
[inst
->dst()].reg(0);
4545 PhysReg destCtxReg
= m_regs
[inst
->dst()].reg(1);
4546 const Class
* cls
= inst
->src(0)->getValClass();
4547 const StringData
* methName
= inst
->src(1)->getValStr();
4548 SSATmp
* srcCtxTmp
= inst
->src(2);
4549 SSATmp
* fp
= inst
->src(3);
4550 PhysReg srcCtxReg
= m_regs
[srcCtxTmp
].reg(0);
4551 Block
* exitLabel
= inst
->taken();
4552 const StringData
* clsName
= cls
->name();
4553 CacheHandle ch
= StaticMethodFCache::alloc(clsName
, methName
,
4554 getContextName(curClass()));
4556 assert(funcDestReg
!= InvalidReg
&& destCtxReg
!= InvalidReg
);
4557 emitMovRegReg(m_as
, srcCtxReg
, destCtxReg
);
4558 m_as
.loadq(rVmTl
[ch
], funcDestReg
);
4559 m_as
.testq(funcDestReg
, funcDestReg
);
4563 // Handle case where method is not entered in the cache
4564 unlikelyIfBlock(CC_E
, [&] (Asm
& a
) {
4565 const Func
* (*lookup
)(CacheHandle
, const Class
*,
4566 const StringData
*, TypedValue
*) = StaticMethodFCache::lookupIR
;
4567 // preserve destCtxReg across the call since it wouldn't be otherwise
4568 RegSet toSave
= m_state
.liveRegs
[inst
] | RegSet(destCtxReg
);
4569 cgCallHelper(a
, Transl::CppCall((TCA
)lookup
),
4570 funcDestReg
, InvalidReg
,
4571 SyncOptions::kSyncPoint
,
4572 ArgGroup(m_regs
).imm(ch
)
4575 .reg(m_regs
[fp
].reg()),
4577 // If entry found in target cache, jump back to m_as.
4578 // Otherwise, bail to exit label
4579 a
.testq(funcDestReg
, funcDestReg
);
4580 emitFwdJcc(a
, CC_Z
, exitLabel
);
4583 auto t
= srcCtxTmp
->type();
4584 assert(!t
.equals(Type::Cls
));
4585 if (t
.equals(Type::Cctx
)) {
4586 return; // done: destCtxReg already has srcCtxReg
4587 } else if (t
== Type::Obj
) {
4588 // unconditionally run code produced by emitGetCtxFwdCallWithThisDyn below
4590 } else if (t
== Type::Ctx
) {
4591 // dynamically check if we have a This pointer and
4592 // call emitGetCtxFwdCallWithThisDyn below
4593 m_as
.testb(1, rbyte(destCtxReg
));
4594 m_as
.jcc8(CC_NZ
, End
);
4599 // If we have a 'this' pointer ...
4600 emitGetCtxFwdCallWithThisDyn(destCtxReg
, destCtxReg
, ch
);
4602 asm_label(m_as
, End
);
4605 void CodeGenerator::cgLdClsPropAddrCached(IRInstruction
* inst
) {
4606 using namespace Transl::TargetCache
;
4607 SSATmp
* dst
= inst
->dst();
4608 SSATmp
* cls
= inst
->src(0);
4609 SSATmp
* propName
= inst
->src(1);
4610 SSATmp
* clsName
= inst
->src(2);
4611 SSATmp
* cxt
= inst
->src(3);
4612 Block
* target
= inst
->taken();
4614 const StringData
* propNameString
= propName
->getValStr();
4615 const StringData
* clsNameString
= clsName
->getValStr();
4617 string
sds(Util::toLower(clsNameString
->data()) + ":" +
4618 string(propNameString
->data(), propNameString
->size()));
4619 StackStringData
sd(sds
.c_str(), sds
.size(), AttachLiteral
);
4620 CacheHandle ch
= SPropCache::alloc(&sd
);
4622 auto dstReg
= m_regs
[dst
].reg();
4623 // Cls is live in the slow path call to lookupIR, so we have to be
4624 // careful not to clobber it before the branch to slow path. So
4625 // use the scratch register as a temporary destination if cls is
4626 // assigned the same register as the dst register.
4627 auto tmpReg
= dstReg
;
4628 if (dstReg
== InvalidReg
|| dstReg
== m_regs
[cls
].reg()) {
4629 tmpReg
= PhysReg(m_rScratch
);
4632 // Could be optimized to cmp against zero when !label && dstReg == InvalidReg
4633 m_as
.loadq(rVmTl
[ch
], tmpReg
);
4634 m_as
.testq(tmpReg
, tmpReg
);
4635 unlikelyIfBlock(CC_E
, [&] (Asm
& a
) {
4637 target
? (TCA
)SPropCache::lookupIR
<false>
4638 : (TCA
)SPropCache::lookupIR
<true>, // raise on error
4640 SyncOptions::kSyncPoint
, // could re-enter to init properties
4641 ArgGroup(m_regs
).imm(ch
).ssa(cls
).ssa(propName
).ssa(cxt
));
4643 a
.testq(tmpReg
, tmpReg
);
4644 emitFwdJcc(a
, CC_Z
, target
);
4647 if (dstReg
!= InvalidReg
) {
4648 emitMovRegReg(m_as
, tmpReg
, dstReg
);
4652 void CodeGenerator::cgLdClsPropAddr(IRInstruction
* inst
) {
4653 SSATmp
* dst
= inst
->dst();
4654 SSATmp
* cls
= inst
->src(0);
4655 SSATmp
* prop
= inst
->src(1);
4656 SSATmp
* ctx
= inst
->src(2);
4657 Block
* target
= inst
->taken();
4658 // If our label is a catch trace we pretend we don't have one, to
4659 // avoid emitting a jmp to it or calling the wrong helper.
4660 if (target
&& target
->trace()->isCatch()) target
= nullptr;
4662 auto dstReg
= m_regs
[dst
].reg();
4663 if (dstReg
== InvalidReg
&& target
) {
4664 // result is unused but this instruction was not eliminated
4665 // because its essential
4666 dstReg
= m_rScratch
;
4669 target
? (TCA
)SPropCache::lookupSProp
<false>
4670 : (TCA
)SPropCache::lookupSProp
<true>, // raise on error
4672 SyncOptions::kSyncPoint
, // could re-enter to init properties
4673 ArgGroup(m_regs
).ssa(cls
).ssa(prop
).ssa(ctx
));
4675 m_as
.testq(dstReg
, dstReg
);
4676 emitFwdJcc(m_as
, CC_Z
, target
);
4680 TargetCache::CacheHandle
CodeGenerator::cgLdClsCachedCommon(
4681 IRInstruction
* inst
) {
4682 SSATmp
* dst
= inst
->dst();
4683 const StringData
* className
= inst
->src(0)->getValStr();
4684 auto ch
= TargetCache::allocKnownClass(className
);
4685 auto dstReg
= m_regs
[dst
].reg();
4686 if (dstReg
== InvalidReg
) {
4687 m_as
. cmpq (0, rVmTl
[ch
]);
4689 m_as
. loadq (rVmTl
[ch
], dstReg
);
4690 m_as
. testq (dstReg
, dstReg
);
4696 void CodeGenerator::cgLdClsCached(IRInstruction
* inst
) {
4697 auto ch
= cgLdClsCachedCommon(inst
);
4698 unlikelyIfBlock(CC_E
, [&] (Asm
& a
) {
4699 // Passing only two arguments to lookupKnownClass, since the
4700 // third is ignored in the checkOnly==false case.
4702 (TCA
)TargetCache::lookupKnownClass
<false>,
4704 SyncOptions::kSyncPoint
,
4705 ArgGroup(m_regs
).addr(rVmTl
, intptr_t(ch
)).ssas(inst
, 0));
4709 void CodeGenerator::cgLdClsCachedSafe(IRInstruction
* inst
) {
4710 cgLdClsCachedCommon(inst
);
4711 if (Block
* taken
= inst
->taken()) {
4712 emitFwdJcc(CC_Z
, taken
);
4716 void CodeGenerator::cgLdCls(IRInstruction
* inst
) {
4717 SSATmp
* dst
= inst
->dst();
4718 SSATmp
* className
= inst
->src(0);
4720 CacheHandle ch
= ClassCache::alloc();
4721 cgCallHelper(m_as
, (TCA
)ClassCache::lookup
, dst
, SyncOptions::kSyncPoint
,
4722 ArgGroup(m_regs
).imm(ch
).ssa(className
));
4725 static StringData
* fullConstName(const StringData
* cls
,
4726 const StringData
* cnsName
) {
4727 return StringData::GetStaticString(
4728 Util::toLower(cls
->data()) + "::" + cnsName
->data()
4732 void CodeGenerator::cgLdClsCns(IRInstruction
* inst
) {
4733 auto const extra
= inst
->extra
<LdClsCns
>();
4734 auto const fullName
= fullConstName(extra
->clsName
, extra
->cnsName
);
4735 auto const ch
= TargetCache::allocClassConstant(fullName
);
4736 cgLoad(rVmTl
, ch
, inst
);
4739 void CodeGenerator::cgLookupClsCns(IRInstruction
* inst
) {
4740 auto const extra
= inst
->extra
<LookupClsCns
>();
4741 auto const fullName
= fullConstName(extra
->clsName
, extra
->cnsName
);
4742 auto const ch
= TargetCache::allocClassConstant(fullName
);
4745 TCA(TargetCache::lookupClassConstantTv
),
4747 SyncOptions::kSyncPoint
,
4750 .immPtr(Unit::GetNamedEntity(extra
->clsName
))
4751 .immPtr(extra
->clsName
)
4752 .immPtr(extra
->cnsName
),
4757 void CodeGenerator::cgLdCns(IRInstruction
* inst
) {
4758 const StringData
* cnsName
= inst
->src(0)->getValStr();
4760 TargetCache::CacheHandle ch
= StringData::DefCnsHandle(cnsName
, false);
4761 // Has an unlikely branch to a LookupCns
4762 cgLoad(rVmTl
, ch
, inst
);
4765 static TypedValue
lookupCnsHelper(const TypedValue
* tv
, StringData
* nm
) {
4766 assert(tv
->m_type
== KindOfUninit
);
4767 TypedValue
*cns
= nullptr;
4769 if (UNLIKELY(tv
->m_data
.pref
!= nullptr)) {
4770 ClassInfo::ConstantInfo
* ci
=
4771 (ClassInfo::ConstantInfo
*)(void*)tv
->m_data
.pref
;
4772 cns
= const_cast<Variant
&>(ci
->getDeferredValue()).asTypedValue();
4773 tvReadCell(cns
, &c1
);
4775 if (UNLIKELY(TargetCache::s_constants
!= nullptr)) {
4776 cns
= TargetCache::s_constants
->HphpArray::nvGet(nm
);
4779 cns
= Unit::loadCns(const_cast<StringData
*>(nm
));
4781 if (UNLIKELY(!cns
)) {
4782 raise_notice(Strings::UNDEFINED_CONSTANT
, nm
->data(), nm
->data());
4783 c1
.m_data
.pstr
= const_cast<StringData
*>(nm
);
4784 c1
.m_type
= KindOfStaticString
;
4786 c1
.m_type
= cns
->m_type
;
4787 c1
.m_data
= cns
->m_data
;
4793 void CodeGenerator::cgLookupCns(IRInstruction
* inst
) {
4794 SSATmp
* cnsNameTmp
= inst
->src(0);
4796 assert(inst
->typeParam() == Type::Cell
);
4797 assert(cnsNameTmp
->isConst() && cnsNameTmp
->type() == Type::StaticStr
);
4799 const StringData
* cnsName
= cnsNameTmp
->getValStr();
4800 TargetCache::CacheHandle ch
= StringData::DefCnsHandle(cnsName
, false);
4802 ArgGroup
args(m_regs
);
4803 args
.addr(rVmTl
, ch
)
4806 cgCallHelper(m_as
, TCA(lookupCnsHelper
),
4807 inst
->dst(), SyncOptions::kSyncPoint
, args
, DestType::TV
);
4811 static inline int64_t ak_exist_string_helper(StringData
* key
, ArrayData
* arr
) {
4813 if (key
->isStrictlyInteger(n
)) {
4814 return arr
->exists(n
);
4816 return arr
->exists(StrNR(key
));
4820 static int64_t ak_exist_string(StringData
* key
, ArrayData
* arr
) {
4821 int64_t res
= ak_exist_string_helper(key
, arr
);
4826 static int64_t ak_exist_int(int64_t key
, ArrayData
* arr
) {
4827 bool res
= arr
->exists(key
);
4832 static int64_t ak_exist_string_obj(StringData
* key
, ObjectData
* obj
) {
4833 if (obj
->isCollection()) {
4834 return collectionOffsetContains(obj
, key
);
4836 CArrRef arr
= obj
->o_toArray();
4837 int64_t res
= ak_exist_string_helper(key
, arr
.get());
4842 static int64_t ak_exist_int_obj(int64_t key
, ObjectData
* obj
) {
4843 if (obj
->isCollection()) {
4844 return collectionOffsetContains(obj
, key
);
4846 CArrRef arr
= obj
->o_toArray();
4847 bool res
= arr
.get()->exists(key
);
4851 void CodeGenerator::cgAKExists(IRInstruction
* inst
) {
4852 SSATmp
* arr
= inst
->src(0);
4853 SSATmp
* key
= inst
->src(1);
4855 if (key
->type().isNull()) {
4856 if (arr
->isA(Type::Arr
)) {
4858 (TCA
)ak_exist_string
,
4860 SyncOptions::kNoSyncPoint
,
4861 ArgGroup(m_regs
).immPtr(empty_string
.get()).ssa(arr
));
4863 m_as
.mov_imm64_reg(0, m_regs
[inst
->dst()].reg());
4870 ? (key
->isA(Type::Int
) ? (TCA
)ak_exist_int_obj
: (TCA
)ak_exist_string_obj
)
4871 : (key
->isA(Type::Int
) ? (TCA
)ak_exist_int
: (TCA
)ak_exist_string
);
4876 SyncOptions::kNoSyncPoint
,
4877 ArgGroup(m_regs
).ssa(key
).ssa(arr
));
4880 HOT_FUNC_VM
static TypedValue
* ldGblAddrHelper(StringData
* name
) {
4881 return g_vmContext
->m_globalVarEnv
->lookup(name
);
4884 HOT_FUNC_VM
static TypedValue
* ldGblAddrDefHelper(StringData
* name
) {
4885 TypedValue
* r
= g_vmContext
->m_globalVarEnv
->lookupAdd(name
);
4890 void CodeGenerator::cgLdGblAddr(IRInstruction
* inst
) {
4891 auto dstReg
= m_regs
[inst
->dst()].reg();
4892 cgCallHelper(m_as
, (TCA
)ldGblAddrHelper
, dstReg
,
4893 SyncOptions::kNoSyncPoint
,
4894 ArgGroup(m_regs
).ssa(inst
->src(0)));
4895 m_as
.testq(dstReg
, dstReg
);
4896 emitFwdJcc(CC_Z
, inst
->taken());
4899 void CodeGenerator::cgLdGblAddrDef(IRInstruction
* inst
) {
4900 cgCallHelper(m_as
, (TCA
)ldGblAddrDefHelper
, inst
->dst(),
4901 SyncOptions::kNoSyncPoint
,
4902 ArgGroup(m_regs
).ssa(inst
->src(0)));
4905 void CodeGenerator::emitTestZero(SSATmp
* src
) {
4907 auto reg
= m_regs
[src
].reg();
4910 * If src is const, normally a earlier optimization pass should have
4911 * converted the thing testing this condition into something
4912 * unconditional. So rather than supporting constants efficiently
4913 * here, we just materialize the value into a register.
4915 if (reg
== InvalidReg
) {
4917 a
. movq (src
->getValBits(), reg
);
4920 if (src
->isA(Type::Bool
)) {
4921 a
. testb (rbyte(reg
), rbyte(reg
));
4923 a
. testq (reg
, reg
);
4927 void CodeGenerator::cgJmpZero(IRInstruction
* inst
) {
4928 emitTestZero(inst
->src(0));
4929 emitFwdJcc(CC_Z
, inst
->taken());
4932 void CodeGenerator::cgJmpNZero(IRInstruction
* inst
) {
4933 emitTestZero(inst
->src(0));
4934 emitFwdJcc(CC_NZ
, inst
->taken());
4937 void CodeGenerator::cgReqBindJmpZero(IRInstruction
* inst
) {
4938 // TODO(#2404427): prepareForTestAndSmash?
4939 emitTestZero(inst
->src(0));
4940 emitReqBindJcc(CC_Z
, inst
->extra
<ReqBindJmpZero
>());
4943 void CodeGenerator::cgReqBindJmpNZero(IRInstruction
* inst
) {
4944 // TODO(#2404427): prepareForTestAndSmash?
4945 emitTestZero(inst
->src(0));
4946 emitReqBindJcc(CC_NZ
, inst
->extra
<ReqBindJmpNZero
>());
4949 void CodeGenerator::cgJmp_(IRInstruction
* inst
) {
4950 Block
* target
= inst
->taken();
4951 if (unsigned n
= inst
->numSrcs()) {
4952 // Parallel-copy sources to the label's destination registers.
4953 // TODO: t2040286: this only works if all destinations fit in registers.
4954 auto srcs
= inst
->srcs();
4955 auto dsts
= target
->front()->dsts();
4956 ArgGroup
args(m_regs
);
4957 for (unsigned i
= 0, j
= 0; i
< n
; i
++) {
4958 assert(srcs
[i
]->type().subtypeOf(dsts
[i
].type()));
4959 auto dst
= &dsts
[i
];
4961 // Currently, full XMM registers cannot be assigned to SSATmps
4962 // passed from to Jmp_ to DefLabel. If this changes, it'll require
4963 // teaching shuffleArgs() how to handle full XMM values.
4964 assert(!m_regs
[src
].isFullXMM() && !m_regs
[dst
].isFullXMM());
4965 if (m_regs
[dst
].reg(0) == InvalidReg
) continue; // dst is unused.
4966 // first dst register
4968 args
[j
++].setDstReg(m_regs
[dst
].reg(0));
4969 // second dst register, if any
4970 if (dst
->numNeededRegs() == 2) {
4971 if (src
->numNeededRegs() < 2) {
4972 // src has known data type, but dst doesn't - pass immediate type
4973 assert(src
->type().isKnownDataType());
4974 args
.imm(src
->type().toDataType());
4976 // pass src's second register
4977 assert(m_regs
[src
].reg(1) != InvalidReg
);
4978 args
.reg(m_regs
[src
].reg(1));
4980 args
[j
++].setDstReg(m_regs
[dst
].reg(1));
4983 assert(args
.numStackArgs() == 0 &&
4984 "Jmp_ doesn't support passing arguments on the stack yet.");
4985 shuffleArgs(m_as
, args
);
4987 if (!m_state
.noTerminalJmp_
) {
4988 emitFwdJmp(m_as
, target
, m_state
);
4992 void CodeGenerator::cgJmpIndirect(IRInstruction
* inst
) {
4993 m_as
.jmp(m_regs
[inst
->src(0)].reg());
4996 void CodeGenerator::cgCheckInit(IRInstruction
* inst
) {
4997 Block
* label
= inst
->taken();
4999 SSATmp
* src
= inst
->src(0);
5001 if (src
->type().isInit()) return;
5003 auto typeReg
= m_regs
[src
].reg(1);
5004 assert(typeReg
!= InvalidReg
);
5006 static_assert(KindOfUninit
== 0, "cgCheckInit assumes KindOfUninit == 0");
5007 m_as
.testb (rbyte(typeReg
), rbyte(typeReg
));
5008 emitFwdJcc(CC_Z
, label
);
5011 void CodeGenerator::cgCheckInitMem(IRInstruction
* inst
) {
5012 Block
* label
= inst
->taken();
5014 SSATmp
* base
= inst
->src(0);
5015 int64_t offset
= inst
->src(1)->getValInt();
5016 Type t
= base
->type().deref();
5017 if (t
.isInit()) return;
5018 auto basereg
= m_regs
[base
].reg();
5019 emitCmpTVType(m_as
, KindOfUninit
, basereg
[offset
+ TVOFF(m_type
)]);
5020 emitFwdJcc(CC_Z
, label
);
5023 void CodeGenerator::cgExitWhenSurprised(IRInstruction
* inst
) {
5024 Block
* label
= inst
->taken();
5025 m_tx64
->emitTestSurpriseFlags(m_as
);
5026 emitFwdJcc(CC_NZ
, label
);
5029 void CodeGenerator::cgExitOnVarEnv(IRInstruction
* inst
) {
5030 SSATmp
* fp
= inst
->src(0);
5031 Block
* label
= inst
->taken();
5033 assert(!(fp
->isConst()));
5035 auto fpReg
= m_regs
[fp
].reg();
5036 m_as
. cmpq (0, fpReg
[AROFF(m_varEnv
)]);
5037 emitFwdJcc(CC_NE
, label
);
5040 void CodeGenerator::cgReleaseVVOrExit(IRInstruction
* inst
) {
5041 auto* const label
= inst
->taken();
5042 auto const rFp
= m_regs
[inst
->src(0)].reg();
5044 m_as
. cmpq (0, rFp
[AROFF(m_varEnv
)]);
5045 unlikelyIfBlock(CC_NZ
, [&] (Asm
& a
) {
5046 a
. testl (ActRec::kExtraArgsBit
, rFp
[AROFF(m_varEnv
)]);
5047 emitFwdJcc(a
, CC_Z
, label
);
5050 TCA(static_cast<void (*)(ActRec
*)>(ExtraArgs::deallocate
)),
5052 SyncOptions::kSyncPoint
,
5053 ArgGroup(m_regs
).reg(rFp
),
5059 void CodeGenerator::cgBoxPtr(IRInstruction
* inst
) {
5060 SSATmp
* dst
= inst
->dst();
5061 SSATmp
* addr
= inst
->src(0);
5062 auto base
= m_regs
[addr
].reg();
5063 auto dstReg
= m_regs
[dst
].reg();
5064 emitMovRegReg(m_as
, base
, dstReg
);
5065 emitTypeTest(Type::BoxedCell
, base
[TVOFF(m_type
)],
5066 base
[TVOFF(m_data
)],
5067 [&](ConditionCode cc
) {
5068 ifThen(m_as
, ccNegate(cc
), [&] {
5069 cgCallHelper(m_as
, (TCA
)tvBox
, dstReg
, SyncOptions::kNoSyncPoint
,
5070 ArgGroup(m_regs
).ssa(addr
));
5075 // TODO: Kill this #2031980
5076 static StringData
* concat_value(TypedValue tv1
, TypedValue tv2
) {
5077 return concat_tv(tv1
.m_type
, tv1
.m_data
.num
, tv2
.m_type
, tv2
.m_data
.num
);
5080 void CodeGenerator::cgConcat(IRInstruction
* inst
) {
5081 SSATmp
* dst
= inst
->dst();
5082 SSATmp
* tl
= inst
->src(0);
5083 SSATmp
* tr
= inst
->src(1);
5085 Type lType
= tl
->type();
5086 Type rType
= tr
->type();
5087 // We have specialized helpers for concatenating two strings, a
5088 // string and an int, and an int and a string.
5089 void* fptr
= nullptr;
5090 if (lType
.isString() && rType
.isString()) {
5091 fptr
= (void*)concat_ss
;
5092 } else if (lType
.isString() && rType
== Type::Int
) {
5093 fptr
= (void*)concat_si
;
5094 } else if (lType
== Type::Int
&& rType
.isString()) {
5095 fptr
= (void*)concat_is
;
5098 cgCallHelper(m_as
, (TCA
)fptr
, dst
, SyncOptions::kNoSyncPoint
,
5099 ArgGroup(m_regs
).ssa(tl
).ssa(tr
));
5101 if (lType
.subtypeOf(Type::Obj
) || lType
.needsReg() ||
5102 rType
.subtypeOf(Type::Obj
) || rType
.needsReg()) {
5105 cgCallHelper(m_as
, (TCA
)concat_value
, dst
, SyncOptions::kNoSyncPoint
,
5106 ArgGroup(m_regs
).typedValue(tl
).typedValue(tr
));
5110 void CodeGenerator::cgInterpOne(IRInstruction
* inst
) {
5111 SSATmp
* fp
= inst
->src(0);
5112 SSATmp
* sp
= inst
->src(1);
5113 auto const& extra
= *inst
->extra
<InterpOne
>();
5114 int64_t pcOff
= extra
.bcOff
;
5116 auto opc
= *(curFunc()->unit()->at(pcOff
));
5117 void* interpOneHelper
= interpOneEntryPoints
[opc
];
5119 auto dstReg
= InvalidReg
;
5120 cgCallHelper(m_as
, (TCA
)interpOneHelper
, dstReg
, SyncOptions::kSyncPoint
,
5121 ArgGroup(m_regs
).ssa(fp
).ssa(sp
).imm(pcOff
));
5123 auto newSpReg
= m_regs
[inst
->dst()].reg();
5124 assert(newSpReg
== m_regs
[sp
].reg());
5126 int64_t spAdjustBytes
= cellsToBytes(extra
.cellsPopped
- extra
.cellsPushed
);
5127 if (spAdjustBytes
!= 0) {
5128 m_as
.addq(spAdjustBytes
, newSpReg
);
5132 void CodeGenerator::cgInterpOneCF(IRInstruction
* inst
) {
5133 SSATmp
* fp
= inst
->src(0);
5134 SSATmp
* sp
= inst
->src(1);
5135 int64_t pcOff
= inst
->src(2)->getValInt();
5137 auto opc
= *(curFunc()->unit()->at(pcOff
));
5138 void* interpOneHelper
= interpOneEntryPoints
[opc
];
5140 auto dstReg
= InvalidReg
;
5141 cgCallHelper(m_as
, (TCA
)interpOneHelper
, dstReg
, SyncOptions::kSyncPoint
,
5142 ArgGroup(m_regs
).ssa(fp
).ssa(sp
).imm(pcOff
));
5144 // The interpOne method returns a pointer to the current ExecutionContext
5145 // in rax. Use it read the 'm_fp' and 'm_stack.m_top' fields into the
5146 // rVmFp and rVmSp registers.
5147 m_as
.loadq(rax
[offsetof(VMExecutionContext
, m_fp
)], rVmFp
);
5148 m_as
.loadq(rax
[offsetof(VMExecutionContext
, m_stack
) +
5149 Stack::topOfStackOffset()], rVmSp
);
5151 m_tx64
->emitServiceReq(SRFlags::EmitInA
, REQ_RESUME
, 0ull);
5154 void CodeGenerator::cgContEnter(IRInstruction
* inst
) {
5155 auto contAR
= inst
->src(0);
5156 auto addr
= inst
->src(1);
5157 auto returnOff
= inst
->src(2);
5158 auto curFp
= m_regs
[inst
->src(3)].reg();
5159 auto contARReg
= m_regs
[contAR
].reg();
5161 m_as
. storel (returnOff
->getValInt(), contARReg
[AROFF(m_soff
)]);
5162 m_as
. storeq (curFp
, contARReg
[AROFF(m_savedRbp
)]);
5163 m_as
. movq (contARReg
, rStashedAR
);
5165 m_as
. call (m_regs
[addr
].reg());
5168 void CodeGenerator::emitContVarEnvHelperCall(SSATmp
* fp
, TCA helper
) {
5169 auto scratch
= m_rScratch
;
5171 m_as
. loadq (m_regs
[fp
].reg()[AROFF(m_varEnv
)], scratch
);
5172 m_as
. testq (scratch
, scratch
);
5173 unlikelyIfBlock(CC_NZ
, [&] (Asm
& a
) {
5174 cgCallHelper(a
, helper
, InvalidReg
, SyncOptions::kNoSyncPoint
,
5175 ArgGroup(m_regs
).ssa(fp
));
5179 void CodeGenerator::cgContPreNext(IRInstruction
* inst
) {
5180 auto contReg
= m_regs
[inst
->src(0)].reg();
5182 const Offset doneOffset
= c_Continuation::doneOffset();
5183 static_assert((doneOffset
+ 1) == c_Continuation::runningOffset(),
5184 "done should immediately precede running");
5185 // Check done and running at the same time
5186 m_as
.testw(0x0101, contReg
[doneOffset
]);
5187 emitFwdJcc(CC_NZ
, inst
->taken());
5190 m_as
.addq(0x1, contReg
[CONTOFF(m_index
)]);
5192 m_as
.storeb(0x1, contReg
[c_Continuation::runningOffset()]);
5195 void CodeGenerator::cgContStartedCheck(IRInstruction
* inst
) {
5196 m_as
.cmp_imm64_disp_reg64(0, CONTOFF(m_index
),
5197 m_regs
[inst
->src(0)].reg());
5198 emitFwdJcc(CC_L
, inst
->taken());
5201 void CodeGenerator::cgIterInit(IRInstruction
* inst
) {
5202 cgIterInitCommon(inst
);
5205 void CodeGenerator::cgIterInitK(IRInstruction
* inst
) {
5206 cgIterInitCommon(inst
);
5209 void CodeGenerator::cgWIterInit(IRInstruction
* inst
) {
5210 cgIterInitCommon(inst
);
5213 void CodeGenerator::cgWIterInitK(IRInstruction
* inst
) {
5214 cgIterInitCommon(inst
);
5217 void CodeGenerator::cgIterInitCommon(IRInstruction
* inst
) {
5218 bool isInitK
= inst
->op() == IterInitK
|| inst
->op() == WIterInitK
;
5219 bool isWInit
= inst
->op() == WIterInit
|| inst
->op() == WIterInitK
;
5221 PhysReg fpReg
= m_regs
[inst
->src(1)].reg();
5222 int64_t iterOffset
= this->iterOffset(inst
->src(2));
5223 int64_t valLocalOffset
= localOffset(inst
->src(3));
5224 SSATmp
* src
= inst
->src(0);
5225 ArgGroup
args(m_regs
);
5226 args
.addr(fpReg
, iterOffset
).ssa(src
);
5227 if (src
->isArray()) {
5228 args
.addr(fpReg
, valLocalOffset
);
5230 args
.addr(fpReg
, localOffset(inst
->src(4)));
5231 } else if (isWInit
) {
5234 TCA helperAddr
= isWInit
? (TCA
)new_iter_array_key
<true> :
5235 isInitK
? (TCA
)new_iter_array_key
<false> : (TCA
)new_iter_array
;
5236 cgCallHelper(m_as
, helperAddr
, inst
->dst(), SyncOptions::kSyncPoint
, args
);
5238 assert(src
->type() == Type::Obj
);
5239 args
.imm(uintptr_t(curClass())).addr(fpReg
, valLocalOffset
);
5241 args
.addr(fpReg
, localOffset(inst
->src(4)));
5245 // new_iter_object decrefs its src object if it propagates an
5246 // exception out, so we use kSyncPointAdjustOne, which adjusts the
5247 // stack pointer by 1 stack element on an unwind, skipping over
5249 cgCallHelper(m_as
, (TCA
)new_iter_object
, inst
->dst(),
5250 SyncOptions::kSyncPointAdjustOne
, args
);
5254 void CodeGenerator::cgIterNext(IRInstruction
* inst
) {
5255 cgIterNextCommon(inst
);
5258 void CodeGenerator::cgIterNextK(IRInstruction
* inst
) {
5259 cgIterNextCommon(inst
);
5262 void CodeGenerator::cgWIterNext(IRInstruction
* inst
) {
5263 cgIterNextCommon(inst
);
5266 void CodeGenerator::cgWIterNextK(IRInstruction
* inst
) {
5267 cgIterNextCommon(inst
);
5270 void CodeGenerator::cgIterNextCommon(IRInstruction
* inst
) {
5271 bool isNextK
= inst
->op() == IterNextK
|| inst
->op() == WIterNextK
;
5272 bool isWNext
= inst
->op() == WIterNext
|| inst
->op() == WIterNextK
;
5273 PhysReg fpReg
= m_regs
[inst
->src(0)].reg();
5274 ArgGroup
args(m_regs
);
5275 args
.addr(fpReg
, iterOffset(inst
->src(1)))
5276 .addr(fpReg
, localOffset(inst
->src(2)));
5278 args
.addr(fpReg
, localOffset(inst
->src(3)));
5279 } else if (isWNext
) {
5282 TCA helperAddr
= isWNext
? (TCA
)iter_next_key
<true> :
5283 isNextK
? (TCA
)iter_next_key
<false> : (TCA
)iter_next
;
5284 cgCallHelper(m_as
, helperAddr
, inst
->dst(), SyncOptions::kSyncPoint
, args
);
5287 void iterFreeHelper(Iter
* iter
) {
5291 void citerFreeHelper(Iter
* iter
) {
5295 void CodeGenerator::cgIterFree(IRInstruction
* inst
) {
5296 PhysReg fpReg
= m_regs
[inst
->src(0)].reg();
5297 int64_t offset
= iterOffset(inst
->extra
<IterFree
>()->iterId
);
5298 cgCallHelper(m_as
, (TCA
)iterFreeHelper
, InvalidReg
, SyncOptions::kSyncPoint
,
5299 ArgGroup(m_regs
).addr(fpReg
, offset
));
5302 void CodeGenerator::cgDecodeCufIter(IRInstruction
* inst
) {
5303 PhysReg fpReg
= m_regs
[inst
->src(1)].reg();
5304 int64_t offset
= iterOffset(inst
->extra
<DecodeCufIter
>()->iterId
);
5305 cgCallHelper(m_as
, (TCA
)decodeCufIterHelper
, inst
->dst(),
5306 SyncOptions::kSyncPoint
,
5307 ArgGroup(m_regs
).addr(fpReg
, offset
).typedValue(inst
->src(0)));
5310 void CodeGenerator::cgCIterFree(IRInstruction
* inst
) {
5311 PhysReg fpReg
= m_regs
[inst
->src(0)].reg();
5312 int64_t offset
= iterOffset(inst
->extra
<CIterFree
>()->iterId
);
5313 cgCallHelper(m_as
, (TCA
)citerFreeHelper
, InvalidReg
, SyncOptions::kSyncPoint
,
5314 ArgGroup(m_regs
).addr(fpReg
, offset
));
5317 void CodeGenerator::cgIncStat(IRInstruction
*inst
) {
5318 Stats::emitInc(m_as
,
5319 Stats::StatCounter(inst
->src(0)->getValInt()),
5320 inst
->src(1)->getValInt(),
5322 inst
->src(2)->getValBool());
5325 void CodeGenerator::cgIncTransCounter(IRInstruction
* inst
) {
5326 m_tx64
->emitTransCounterInc(m_as
);
5329 void CodeGenerator::cgDbgAssertRefCount(IRInstruction
* inst
) {
5330 emitAssertRefCount(m_as
, m_regs
[inst
->src(0)].reg());
5333 void traceCallback(ActRec
* fp
, Cell
* sp
, int64_t pcOff
, void* rip
) {
5334 if (HPHP::Trace::moduleEnabled(HPHP::Trace::hhirTracelets
)) {
5335 FTRACE(0, "{} {} {}\n", fp
->m_func
->fullName()->data(), pcOff
, rip
);
5337 checkFrame(fp
, sp
, /*checkLocals*/true);
5340 void CodeGenerator::cgDbgAssertType(IRInstruction
* inst
) {
5341 emitTypeTest(inst
->typeParam(),
5342 m_regs
[inst
->src(0)].reg(1),
5343 m_regs
[inst
->src(0)].reg(0),
5344 [&](ConditionCode cc
) {
5345 ifThen(m_as
, ccNegate(cc
), [&] { m_as
.ud2(); });
5349 void CodeGenerator::cgVerifyParamCls(IRInstruction
* inst
) {
5350 SSATmp
* objClass
= inst
->src(0);
5351 assert(!objClass
->isConst());
5352 auto objClassReg
= m_regs
[objClass
].reg();
5353 SSATmp
* constraint
= inst
->src(1);
5355 if (constraint
->isConst()) {
5356 m_as
. cmpq(constraint
->getValClass(), objClassReg
);
5358 m_as
. cmpq(m_regs
[constraint
].reg(), objClassReg
);
5361 // The native call for this instruction is the slow path that does
5362 // proper subtype checking. The comparison above is just to
5363 // short-circuit the overhead when the Classes are an exact match.
5364 ifThen(m_as
, CC_NE
, [&]{ cgCallNative(inst
); });
5367 static void emitTraceCall(CodeGenerator::Asm
& as
,
5369 Transl::TranslatorX64
* tx64
) {
5370 // call to a trace function
5371 as
.mov_imm64_reg((int64_t)as
.code
.frontier
, reg::rcx
);
5372 as
.mov_reg64_reg64(rVmFp
, reg::rdi
);
5373 as
.mov_reg64_reg64(rVmSp
, reg::rsi
);
5374 as
.mov_imm64_reg(pcOff
, reg::rdx
);
5375 // do the call; may use a trampoline
5376 tx64
->emitCall(as
, (TCA
)traceCallback
);
5379 void CodeGenerator::print() const {
5380 JIT::print(std::cout
, m_curTrace
, &m_state
.regs
, m_state
.lifetime
,
5384 static void patchJumps(Asm
& as
, CodegenState
& state
, Block
* block
) {
5385 void* list
= state
.patches
[block
];
5386 Address labelAddr
= as
.code
.frontier
;
5388 int32_t* toPatch
= (int32_t*)list
;
5389 int32_t diffToNext
= *toPatch
;
5390 ssize_t diff
= labelAddr
- ((Address
)list
+ sizeof(int32_t));
5391 *toPatch
= safe_cast
<int32_t>(diff
); // patch the jump address
5392 if (diffToNext
== 0) break;
5393 void* next
= (TCA
)list
- diffToNext
;
5398 void CodeGenerator::cgBlock(Block
* block
, vector
<TransBCMapping
>* bcMap
) {
5399 FTRACE(6, "cgBlock: {}\n", block
->id());
5401 BCMarker prevMarker
;
5402 for (IRInstruction
& instr
: *block
) {
5403 IRInstruction
* inst
= &instr
;
5404 // If we're on the first instruction of the block or we have a new
5405 // marker since the last instruction, update the bc mapping.
5406 if ((!prevMarker
.valid() || inst
->marker() != prevMarker
) &&
5407 m_tx64
->isTransDBEnabled() && bcMap
) {
5408 bcMap
->push_back(TransBCMapping
{inst
->marker().bcOff
,
5410 m_astubs
.code
.frontier
});
5411 prevMarker
= inst
->marker();
5414 auto nuller
= folly::makeGuard([&]{ m_curInst
= nullptr; });
5415 auto* addr
= cgInst(inst
);
5416 if (m_state
.asmInfo
&& addr
) {
5417 m_state
.asmInfo
->instRanges
[inst
] = TcaRange(addr
, m_as
.code
.frontier
);
5418 m_state
.asmInfo
->asmRanges
[block
] =
5419 TcaRange(m_state
.asmInfo
->asmRanges
[block
].start(), m_as
.code
.frontier
);
5425 * Compute and save registers that are live *across* each inst, not including
5426 * registers whose lifetimes end at inst, nor registers defined by inst.
5428 LiveRegs
computeLiveRegs(const IRFactory
* factory
, const RegAllocInfo
& regs
,
5429 Block
* start_block
) {
5430 StateVector
<Block
, RegSet
> liveMap(factory
, RegSet());
5431 LiveRegs
live_regs(factory
, RegSet());
5434 RegSet
& live
= liveMap
[block
];
5435 if (Block
* taken
= block
->taken()) live
= liveMap
[taken
];
5436 if (Block
* next
= block
->next()) live
|= liveMap
[next
];
5437 for (auto it
= block
->end(); it
!= block
->begin(); ) {
5438 IRInstruction
& inst
= *--it
;
5439 for (const SSATmp
& dst
: inst
.dsts()) {
5440 live
-= regs
[dst
].regs();
5442 live_regs
[inst
] = live
;
5443 for (SSATmp
* src
: inst
.srcs()) {
5444 live
|= regs
[src
].regs();
5448 factory
->numBlocks(),
5454 void genCodeForTrace(IRTrace
* trace
,
5455 CodeGenerator::Asm
& as
,
5456 CodeGenerator::Asm
& astubs
,
5457 IRFactory
* irFactory
,
5458 vector
<TransBCMapping
>* bcMap
,
5459 Transl::TranslatorX64
* tx64
,
5460 const RegAllocInfo
& regs
,
5461 const LifetimeInfo
* lifetime
,
5463 assert(trace
->isMain());
5464 LiveRegs live_regs
= computeLiveRegs(irFactory
, regs
, trace
->front());
5465 CodegenState
state(irFactory
, regs
, live_regs
, lifetime
, asmInfo
);
5467 // Returns: whether a block has already been emitted.
5468 DEBUG_ONLY
auto isEmitted
= [&](Block
* block
) {
5469 return state
.addresses
[block
];
5473 * Emit the given block on the supplied assembler. The `nextBlock'
5474 * is the nextBlock that will be emitted on this assembler. If is
5475 * not the fallthrough block, emit a patchable jump to the
5476 * fallthrough block.
5478 auto emitBlock
= [&](Asm
& a
, Block
* block
, Block
* nextBlock
) {
5479 assert(!isEmitted(block
));
5481 FTRACE(6, "cgBlock {} on {}\n", block
->id(),
5482 &a
== &astubs
? "astubs" : "a");
5484 auto const aStart
= a
.code
.frontier
;
5485 auto const astubsStart
= astubs
.code
.frontier
;
5486 patchJumps(a
, state
, block
);
5487 state
.addresses
[block
] = aStart
;
5489 // If the block ends with a Jmp_ and the next block is going to be
5490 // its target, we don't need to actually emit it.
5491 IRInstruction
* last
= block
->back();
5492 state
.noTerminalJmp_
= last
->op() == Jmp_
&& nextBlock
== last
->taken();
5494 CodeGenerator
cg(trace
, a
, astubs
, tx64
, state
);
5495 if (state
.asmInfo
) {
5496 state
.asmInfo
->asmRanges
[block
] = TcaRange(aStart
, a
.code
.frontier
);
5499 cg
.cgBlock(block
, bcMap
);
5500 if (auto next
= block
->next()) {
5501 if (next
!= nextBlock
) {
5502 // If there's a fallthrough block and it's not the next thing
5503 // going into this assembler, then emit a jump to it.
5504 emitFwdJmp(a
, next
, state
);
5508 if (state
.asmInfo
) {
5509 state
.asmInfo
->asmRanges
[block
] = TcaRange(aStart
, a
.code
.frontier
);
5510 if (&a
!= &astubs
) {
5511 state
.asmInfo
->astubRanges
[block
] = TcaRange(astubsStart
,
5512 astubs
.code
.frontier
);
5517 if (RuntimeOption::EvalHHIRGenerateAsserts
&& trace
->isMain()) {
5518 emitTraceCall(as
, trace
->bcOff(), tx64
);
5521 auto const linfo
= layoutBlocks(trace
, *irFactory
);
5523 for (auto it
= linfo
.blocks
.begin(); it
!= linfo
.astubsIt
; ++it
) {
5524 Block
* nextBlock
= boost::next(it
) != linfo
.astubsIt
5525 ? *boost::next(it
) : nullptr;
5526 emitBlock(as
, *it
, nextBlock
);
5528 for (auto it
= linfo
.astubsIt
; it
!= linfo
.blocks
.end(); ++it
) {
5529 Block
* nextBlock
= boost::next(it
) != linfo
.blocks
.end()
5530 ? *boost::next(it
) : nullptr;
5531 emitBlock(astubs
, *it
, nextBlock
);
5535 for (Block
* UNUSED block
: linfo
.blocks
) {
5536 assert(isEmitted(block
));
5541 inline ALWAYS_INLINE
5542 TypedValue
& getDefaultIfNullCell(TypedValue
* tv
, TypedValue
& def
) {
5543 if (UNLIKELY(nullptr == tv
)) {
5544 // refcount is already correct since def was never decrefed
5547 tvRefcountedDecRef(&def
);
5548 TypedValue
* ret
= tvToCell(tv
);
5549 tvRefcountedIncRef(ret
);
5554 TypedValue
arrayIdxS(ArrayData
* a
, StringData
* key
, TypedValue def
) {
5555 return getDefaultIfNullCell(a
->nvGet(key
), def
);
5559 TypedValue
arrayIdxSi(ArrayData
* a
, StringData
* key
, TypedValue def
) {
5561 return UNLIKELY(key
->isStrictlyInteger(i
)) ?
5562 getDefaultIfNullCell(a
->nvGet(i
), def
) :
5563 getDefaultIfNullCell(a
->nvGet(key
), def
);
5567 TypedValue
arrayIdxI(ArrayData
* a
, int64_t key
, TypedValue def
) {
5568 return getDefaultIfNullCell(a
->nvGet(key
), def
);