?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Inline object constructors
[hiphop-php.git] / hphp / runtime / vm / translator / hopt / codegen.cpp
blob64df8363aab5f60e338897ede851bd8616b6521e
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010- Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 */
16
17 #include "runtime/vm/translator/hopt/codegen.h"
18
19 #include <string.h>
20
21 #include "folly/ScopeGuard.h"
22 #include "util/trace.h"
23 #include "util/util.h"
24
25 #include "runtime/base/array/hphp_array.h"
26 #include "runtime/base/comparisons.h"
27 #include "runtime/base/complex_types.h"
28 #include "runtime/base/runtime_option.h"
29 #include "runtime/base/string_data.h"
30 #include "runtime/base/types.h"
31 #include "runtime/ext/ext_closure.h"
32 #include "runtime/ext/ext_continuation.h"
33 #include "runtime/ext/ext_collections.h"
34 #include "runtime/vm/bytecode.h"
35 #include "runtime/vm/runtime.h"
36 #include "runtime/base/stats.h"
37 #include "runtime/vm/translator/targetcache.h"
38 #include "runtime/vm/translator/translator-inline.h"
39 #include "runtime/vm/translator/translator-x64.h"
40 #include "runtime/vm/translator/translator-x64-internal.h"
41 #include "runtime/vm/translator/translator.h"
42 #include "runtime/vm/translator/types.h"
43 #include "runtime/vm/translator/x64-util.h"
44 #include "runtime/vm/translator/hopt/ir.h"
45 #include "runtime/vm/translator/hopt/linearscan.h"
46 #include "runtime/vm/translator/hopt/nativecalls.h"
47 #include "runtime/vm/translator/hopt/print.h"
48
49 using HPHP::VM::Transl::TCA;
50 using namespace HPHP::VM::Transl::TargetCache;
51
52 namespace HPHP {
53 namespace VM {
54 namespace JIT {
55
56 namespace {
57
58 //////////////////////////////////////////////////////////////////////
59
60 using namespace Util;
61 using namespace Transl::reg;
62
63 static const HPHP::Trace::Module TRACEMOD = HPHP::Trace::hhir;
64
65 /*
66 * It's not normally ok to directly use tracelet abi registers in
67 * codegen, unless you're directly dealing with an instruction that
68 * does near-end-of-tracelet glue. (Or also we sometimes use them
69 * just for some static_assertions relating to calls to helpers from
70 * tx64 that hardcode these registers.)
71 */
72 using Transl::rVmFp;
73 using Transl::rVmSp;
74
75 const size_t kTypeShiftBits = (offsetof(TypedValue, m_type) % 8) * 8;
76
77 // left shift an immediate DataType, for type, to the correct position
78 // within one of the registers used to pass a TypedValue by value.
79 uint64_t toDataTypeForCall(Type type) {
80 return uint64_t(type.toDataType()) << kTypeShiftBits;
81 }
82
83 int64_t spillSlotsToSize(int n) {
84 return n * sizeof(int64_t);
85 }
86
87 void cgPunt(const char* file, int line, const char* func) {
88 if (dumpIREnabled()) {
89 HPHP::Trace::trace("--------- CG_PUNT %s %d %s\n", file, line, func);
90 }
91 throw FailedCodeGen(file, line, func);
92 }
93
94 #define CG_PUNT(instr) cgPunt(__FILE__, __LINE__, #instr)
95
96 struct CycleInfo {
97 int node;
98 int length;
99 };
100
101 struct MoveInfo {
102 enum Kind { Move, Xchg };
103
104 MoveInfo(Kind kind, int reg1, int reg2):
105 m_kind(kind), m_reg1(reg1), m_reg2(reg2) {}
106
107 Kind m_kind;
108 PhysReg m_reg1, m_reg2;
109 };
110
111 template <int N>
112 void doRegMoves(int (&moves)[N], int rTmp,
113 std::vector<MoveInfo>& howTo) {
114 assert(howTo.empty());
115 int outDegree[N];
116 CycleInfo cycles[N];
117 int numCycles = 0;
118 // Iterate over the nodes filling in outDegree[] and cycles[] as we go
119 {
120 int index[N];
121 for (int node = 0; node < N; ++node) {
122 // If a node's source is itself, its a nop
123 if (moves[node] == node) moves[node] = -1;
124 if (node == rTmp && moves[node] >= 0) {
125 // ERROR: rTmp cannot be referenced in moves[].
126 assert(false);
127 }
128 outDegree[node] = 0;
129 index[node] = -1;
130 }
131 int nextIndex = 0;
132 for (int startNode = 0; startNode < N; ++startNode) {
133 // If startNode has not been visited yet, begin walking
134 // a path from start node
135 if (index[startNode] < 0) {
136 int node = startNode;
137 pathloop:
138 index[node] = nextIndex++;
139 if (moves[node] >= 0) {
140 int nextNode = moves[node];
141 ++outDegree[nextNode];
142 if (index[nextNode] < 0) {
143 // If there is an edge from v to nextNode and nextNode has not been
144 // visited, extend the current path to include nextNode and recurse
145 node = nextNode;
146 goto pathloop;
147 }
148 // There is an edge from v to nextNode but nextNode has already been
149 // visited, check if nextNode is on the current path
150 if (index[nextNode] >= index[startNode]) {
151 // nextNode is on the current path so we've found a cycle
152 int length = nextIndex - index[nextNode];
153 CycleInfo ci = { nextNode, length };
154 cycles[numCycles] = ci;
155 ++numCycles;
156 }
157 }
158 }
159 }
160 }
161 // Handle all moves that aren't part of a cycle
162 {
163 int q[N];
164 int qBack = 0;
165 for (int node = 0; node < N; ++node) {
166 if (outDegree[node] == 0) {
167 q[qBack] = node;
168 ++qBack;
169 }
170 }
171 for (int i = 0; i < qBack; ++i) {
172 int node = q[i];
173 if (moves[node] >= 0) {
174 int nextNode = moves[node];
175 howTo.push_back(MoveInfo(MoveInfo::Move, nextNode, node));
176 --outDegree[nextNode];
177 if (outDegree[nextNode] == 0) {
178 q[qBack] = nextNode;
179 ++qBack;
180 }
181 }
182 }
183 }
184 // Deal with any cycles we encountered
185 for (int i = 0; i < numCycles; ++i) {
186 if (cycles[i].length == 2) {
187 int v = cycles[i].node;
188 int w = moves[v];
189 howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
190 } else if (cycles[i].length == 3) {
191 int v = cycles[i].node;
192 int w = moves[v];
193 howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
194 int x = moves[w];
195 howTo.push_back(MoveInfo(MoveInfo::Xchg, x, w));
196 } else {
197 int v = cycles[i].node;
198 howTo.push_back(MoveInfo(MoveInfo::Move, v, rTmp));
199 int w = v;
200 int x = moves[w];
201 while (x != v) {
202 howTo.push_back(MoveInfo(MoveInfo::Move, x, w));
203 w = x;
204 x = moves[w];
205 }
206 howTo.push_back(MoveInfo(MoveInfo::Move, rTmp, w));
207 }
208 }
209 }
210
211 const char* getContextName(Class* ctx) {
212 return ctx ? ctx->name()->data() : ":anonymous:";
213 }
214
215 } // unnamed namespace
216
217 //////////////////////////////////////////////////////////////////////
218
219 ArgDesc::ArgDesc(SSATmp* tmp, bool val) : m_imm(-1), m_zeroExtend(false),
220 m_done(false) {
221 if (tmp->type() == Type::None) {
222 assert(val);
223 m_kind = None;
224 return;
225 }
226 if (tmp->inst()->op() == DefConst) {
227 m_srcReg = InvalidReg;
228 if (val) {
229 m_imm = tmp->getValBits();
230 } else {
231 m_imm = toDataTypeForCall(tmp->type());
232 }
233 m_kind = Imm;
234 return;
235 }
236 if (tmp->type().isNull()) {
237 m_srcReg = InvalidReg;
238 if (val) {
239 m_imm = 0;
240 } else {
241 m_imm = toDataTypeForCall(tmp->type());
242 }
243 m_kind = Imm;
244 return;
245 }
246 if (val || tmp->numNeededRegs() > 1) {
247 auto reg = tmp->getReg(val ? 0 : 1);
248 assert(reg != InvalidReg);
249 m_imm = 0;
250
251 // If val is false then we're passing tmp's type. TypeReg lets
252 // CodeGenerator know that the value might require some massaging
253 // to be in the right format for the call.
254 m_kind = val ? Reg : TypeReg;
255 // zero extend any boolean value that we pass to the helper in case
256 // the helper expects it (e.g., as TypedValue)
257 if (val && tmp->isA(Type::Bool)) m_zeroExtend = true;
258 m_srcReg = reg;
259 return;
260 }
261 m_srcReg = InvalidReg;
262 m_imm = toDataTypeForCall(tmp->type());
263 m_kind = Imm;
264 }
265
266 const Func* CodeGenerator::getCurFunc() const {
267 always_assert(m_state.lastMarker &&
268 "We shouldn't be looking for a func when we have no marker");
269 return m_state.lastMarker->func;
270 }
271
272 Address CodeGenerator::cgInst(IRInstruction* inst) {
273 Opcode opc = inst->op();
274 auto const start = m_as.code.frontier;
275 switch (opc) {
276 #define O(name, dsts, srcs, flags) \
277 case name: cg ## name (inst); \
278 return m_as.code.frontier == start ? nullptr : start;
279 IR_OPCODES
280 #undef O
281
282 default:
283 assert(0);
284 return nullptr;
285 }
286 }
287
288 #define NOOP_OPCODE(opcode) \
289 void CodeGenerator::cg##opcode(IRInstruction*) {}
290
291 #define PUNT_OPCODE(opcode) \
292 void CodeGenerator::cg##opcode(IRInstruction*) { CG_PUNT(opcode); }
293
294 #define CALL_OPCODE(opcode) \
295 void CodeGenerator::cg##opcode(IRInstruction* i) { cgCallNative(i); }
296
297 #define CALL_STK_OPCODE(opcode) \
298 CALL_OPCODE(opcode) \
299 CALL_OPCODE(opcode ## Stk)
300
301 PUNT_OPCODE(DefCls)
302
303 NOOP_OPCODE(DefConst)
304 NOOP_OPCODE(DefFP)
305 NOOP_OPCODE(DefSP)
306 NOOP_OPCODE(Marker)
307 NOOP_OPCODE(AssertLoc)
308 NOOP_OPCODE(OverrideLoc)
309 NOOP_OPCODE(AssertStk)
310 NOOP_OPCODE(Nop)
311 NOOP_OPCODE(DefLabel)
312 NOOP_OPCODE(ExceptionBarrier)
313
314 CALL_OPCODE(AddElemStrKey)
315 CALL_OPCODE(AddElemIntKey)
316 CALL_OPCODE(AddNewElem)
317 CALL_OPCODE(ArrayAdd)
318 CALL_OPCODE(Box)
319
320 CALL_OPCODE(ConvBoolToArr);
321 CALL_OPCODE(ConvDblToArr);
322 CALL_OPCODE(ConvIntToArr);
323 CALL_OPCODE(ConvObjToArr);
324 CALL_OPCODE(ConvStrToArr);
325 CALL_OPCODE(ConvCellToArr);
326
327 CALL_OPCODE(ConvArrToBool);
328 CALL_OPCODE(ConvStrToBool);
329 CALL_OPCODE(ConvCellToBool);
330
331 CALL_OPCODE(ConvArrToDbl);
332 CALL_OPCODE(ConvObjToDbl);
333 CALL_OPCODE(ConvStrToDbl);
334 CALL_OPCODE(ConvCellToDbl);
335
336 CALL_OPCODE(ConvArrToInt);
337 CALL_OPCODE(ConvDblToInt);
338 CALL_OPCODE(ConvObjToInt);
339 CALL_OPCODE(ConvStrToInt);
340 CALL_OPCODE(ConvCellToInt);
341
342 CALL_OPCODE(ConvCellToObj);
343
344 CALL_OPCODE(ConvDblToStr);
345 CALL_OPCODE(ConvIntToStr);
346 CALL_OPCODE(ConvObjToStr);
347 CALL_OPCODE(ConvCellToStr);
348
349 CALL_OPCODE(CreateCont)
350 CALL_OPCODE(FillContLocals)
351 CALL_OPCODE(NewArray)
352 CALL_OPCODE(NewTuple)
353 CALL_OPCODE(NewObj)
354 CALL_OPCODE(LdClsCtor);
355 CALL_OPCODE(CreateCl)
356 CALL_OPCODE(PrintStr)
357 CALL_OPCODE(PrintInt)
358 CALL_OPCODE(PrintBool)
359 CALL_OPCODE(DbgAssertPtr)
360 CALL_OPCODE(LdSwitchDblIndex)
361 CALL_OPCODE(LdSwitchStrIndex)
362 CALL_OPCODE(LdSwitchObjIndex)
363 CALL_OPCODE(VerifyParamCallable)
364 CALL_OPCODE(VerifyParamFail)
365 CALL_OPCODE(RaiseUninitLoc)
366 CALL_OPCODE(WarnNonObjProp)
367 CALL_OPCODE(ThrowNonObjProp)
368 CALL_OPCODE(RaiseUndefProp)
369 CALL_OPCODE(RaiseError)
370 CALL_OPCODE(RaiseWarning)
371 CALL_OPCODE(IncStatGrouped)
372 CALL_OPCODE(StaticLocInit)
373 CALL_OPCODE(StaticLocInitCached)
374 CALL_OPCODE(OpMod)
375
376 // Vector instruction helpers
377 CALL_OPCODE(BaseG)
378 CALL_OPCODE(PropX)
379 CALL_STK_OPCODE(PropDX)
380 CALL_OPCODE(CGetProp)
381 CALL_STK_OPCODE(VGetProp)
382 CALL_STK_OPCODE(BindProp)
383 CALL_STK_OPCODE(SetProp)
384 CALL_OPCODE(UnsetProp)
385 CALL_STK_OPCODE(SetOpProp)
386 CALL_STK_OPCODE(IncDecProp)
387 CALL_OPCODE(EmptyProp)
388 CALL_OPCODE(IssetProp)
389 CALL_OPCODE(ElemX)
390 CALL_STK_OPCODE(ElemDX)
391 CALL_STK_OPCODE(ElemUX)
392 CALL_OPCODE(ArrayGet)
393 CALL_OPCODE(CGetElem)
394 CALL_STK_OPCODE(VGetElem)
395 CALL_STK_OPCODE(BindElem)
396 CALL_OPCODE(ArraySet)
397 CALL_OPCODE(ArraySetRef)
398 CALL_STK_OPCODE(SetElem)
399 CALL_STK_OPCODE(UnsetElem)
400 CALL_STK_OPCODE(SetOpElem)
401 CALL_STK_OPCODE(IncDecElem)
402 CALL_STK_OPCODE(SetNewElem)
403 CALL_STK_OPCODE(BindNewElem)
404 CALL_OPCODE(ArrayIsset)
405 CALL_OPCODE(IssetElem)
406 CALL_OPCODE(EmptyElem)
407
408 #undef NOOP_OPCODE
409 #undef PUNT_OPCODE
410
411 // Thread chain of patch locations using the 4 byte space in each jmp/jcc
412 void prependPatchAddr(CodegenState& state, Block* block, TCA patchAddr) {
413 auto &patches = state.patches;
414 ssize_t diff = patches[block] ? (patchAddr - (TCA)patches[block]) : 0;
415 assert(deltaFits(diff, sz::dword));
416 *(int32_t*)(patchAddr) = (int32_t)diff;
417 patches[block] = patchAddr;
418 }
419
420 Address CodeGenerator::emitFwdJcc(Asm& a, ConditionCode cc, Block* target) {
421 assert(target);
422 Address start = a.code.frontier;
423 a.jcc(cc, a.code.frontier);
424 TCA immPtr = a.code.frontier - 4;
425 prependPatchAddr(m_state, target, immPtr);
426 return start;
427 }
428
429 Address CodeGenerator::emitFwdJmp(Asm& a, Block* target, CodegenState& state) {
430 Address start = a.code.frontier;
431 a.jmp(a.code.frontier);
432 TCA immPtr = a.code.frontier - 4;
433 prependPatchAddr(state, target, immPtr);
434 return start;
435 }
436
437 Address CodeGenerator::emitFwdJmp(Asm& a, Block* target) {
438 return emitFwdJmp(a, target, m_state);
439 }
440
441 Address CodeGenerator::emitFwdJcc(ConditionCode cc, Block* target) {
442 return emitFwdJcc(m_as, cc, target);
443 }
444
445 Address CodeGenerator::emitFwdJmp(Block* target) {
446 return emitFwdJmp(m_as, target);
447 }
448
449 // Patch with service request REQ_BIND_JMPCC_FIRST/SECOND
450 Address CodeGenerator::emitSmashableFwdJccAtEnd(ConditionCode cc, Block* target,
451 IRInstruction* toSmash) {
452 Address start = m_as.code.frontier;
453 if (toSmash) {
454 m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpLen +
455 TranslatorX64::kJmpccLen);
456 Address tcaJcc = emitFwdJcc(cc, target);
457 emitFwdJmp(target);
458 toSmash->setTCA(tcaJcc);
459 } else {
460 emitFwdJcc(cc, target);
461 }
462 return start;
463 }
464
465 void CodeGenerator::emitJccDirectExit(IRInstruction* inst,
466 ConditionCode cc) {
467 if (cc == CC_None) return;
468 auto* toSmash = inst->getTCA() == kIRDirectJccJmpActive ? inst : nullptr;
469 emitSmashableFwdJccAtEnd(cc, inst->getTaken(), toSmash);
470 }
471
472 // Patch with service request REQ_BIND_JCC
473 Address CodeGenerator::emitSmashableFwdJcc(ConditionCode cc, Block* target,
474 IRInstruction* toSmash) {
475 Address start = m_as.code.frontier;
476 assert(toSmash);
477
478 m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen);
479 Address tcaJcc = emitFwdJcc(cc, target);
480 toSmash->setTCA(tcaJcc);
481 return start;
482 }
483
484 void emitLoadImm(CodeGenerator::Asm& as, int64_t val, PhysReg dstReg) {
485 as.emitImmReg(val, dstReg);
486 }
487
488 static void
489 emitMovRegReg(CodeGenerator::Asm& as, PhysReg srcReg, PhysReg dstReg) {
490 if (srcReg != dstReg) as.movq(srcReg, dstReg);
491 }
492
493 static void emitLea(CodeGenerator::Asm& as, MemoryRef mr, PhysReg dst) {
494 if (dst == InvalidReg) return;
495 if (mr.r.disp == 0) {
496 emitMovRegReg(as, mr.r.base, dst);
497 } else {
498 as.lea(mr, dst);
499 }
500 }
501
502 void shuffle2(CodeGenerator::Asm& a,
503 PhysReg s0, PhysReg s1, PhysReg d0, PhysReg d1) {
504 assert(s0 != s1);
505 if (d0 == s1 && d1 != InvalidReg) {
506 assert(d0 != d1);
507 if (d1 == s0) {
508 a. xchgq (s1, s0);
509 } else {
510 a. movq (s1, d1); // save s1 first; d1 != s0
511 a. movq (s0, d0);
512 }
513 } else {
514 if (d0 != InvalidReg) emitMovRegReg(a, s0, d0); // d0 != s1
515 if (d1 != InvalidReg) emitMovRegReg(a, s1, d1);
516 }
517 }
518
519 static void signExtendBool(X64Assembler& as, const SSATmp* src) {
520 auto reg = src->getReg();
521 if (reg != InvalidReg) {
522 // sign-extend the bool from a byte to a quad
523 as.movsbq(rbyte(reg), r64(reg));
524 }
525 }
526
527 static void zeroExtendIfBool(X64Assembler& as, const SSATmp* src) {
528 if (src->isA(Type::Bool)) {
529 auto reg = src->getReg();
530 if (reg != InvalidReg) {
531 // zero-extend the bool from a byte to a quad
532 // note: movzbl actually extends the value to 64 bits.
533 as.movzbl(rbyte(reg), r32(reg));
534 }
535 }
536 }
537
538 static void prepUnaryXmmOp(X64Assembler& a, const SSATmp* ssa, RegXMM xmm) {
539 RegNumber src(ssa->getReg());
540 if (ssa->getReg() == InvalidReg) {
541 src = rScratch;
542 assert(ssa->isConst());
543 a.mov_imm64_reg(ssa->getValBits(), rScratch);
544 }
545 if (ssa->isA(Type::Int | Type::Bool)) {
546 // Expand non-const bools to 64-bit.
547 // Consts are already moved into src as 64-bit values above.
548 if (!ssa->isConst()) zeroExtendIfBool(a, ssa);
549 // cvtsi2sd doesn't modify the high bits of its target, which can
550 // cause false dependencies to prevent register renaming from kicking
551 // in. Break the dependency chain by zeroing out the destination reg.
552 a. pxor_xmm_xmm(xmm, xmm);
553 a. cvtsi2sd_reg64_xmm(src, xmm);
554 } else {
555 a. mov_reg64_xmm(src, xmm);
556 }
557 }
558
559 static void prepBinaryXmmOp(X64Assembler& a, const SSATmp* left,
560 const SSATmp* right) {
561 prepUnaryXmmOp(a, left, xmm0);
562 prepUnaryXmmOp(a, right, xmm1);
563 }
564
565 static void doubleCmp(X64Assembler& a, RegXMM xmm0, RegXMM xmm1) {
566 a. ucomisd_xmm_xmm(xmm0, xmm1);
567 Label notPF;
568 a. jnp8(notPF);
569 // PF means the doubles were unordered. We treat this as !equal, so
570 // clear ZF.
571 a. or_imm32_reg64(1, rScratch);
572 asm_label(a, notPF);
573 }
574
575 void CodeGenerator::cgJcc(IRInstruction* inst) {
576 SSATmp* src1 = inst->getSrc(0);
577 SSATmp* src2 = inst->getSrc(1);
578 Opcode opc = inst->op();
579 ConditionCode cc = queryJmpToCC(opc);
580 Type src1Type = src1->type();
581 Type src2Type = src2->type();
582
583 // can't generate CMP instructions correctly for anything that isn't
584 // a bool or a numeric, and we can't mix bool/numerics because
585 // -1 == true in PHP, but not in HHIR binary representation
586 if (!((src1Type == Type::Int && src2Type == Type::Int) ||
587 ((src1Type == Type::Int || src1Type == Type::Dbl) &&
588 (src2Type == Type::Int || src2Type == Type::Dbl)) ||
589 (src1Type == Type::Bool && src2Type == Type::Bool) ||
590 (src1Type == Type::Cls && src2Type == Type::Cls))) {
591 CG_PUNT(cgJcc);
592 }
593 if (src1Type == Type::Dbl || src2Type == Type::Dbl) {
594 prepBinaryXmmOp(m_as, src1, src2);
595 doubleCmp(m_as, xmm0, xmm1);
596 } else {
597 if (src1Type == Type::Cls && src2Type == Type::Cls) {
598 assert(opc == JmpSame || opc == JmpNSame);
599 }
600 auto srcReg1 = src1->getReg();
601 auto srcReg2 = src2->getReg();
602
603 // Note: when both src1 and src2 are constants, we should transform the
604 // branch into an unconditional jump earlier in the IR.
605 if (src1->isConst()) {
606 // TODO: use compare with immediate or make sure simplifier
607 // canonicalizes this so that constant is src2
608 srcReg1 = rScratch;
609 m_as. mov_imm64_reg(src1->getValRawInt(), srcReg1);
610 }
611 if (src2->isConst()) {
612 if (src1Type.subtypeOf(Type::Bool)) {
613 m_as. cmpb (src2->getValRawInt(), Reg8(int(srcReg1)));
614 } else {
615 m_as. cmp_imm64_reg64(src2->getValRawInt(), srcReg1);
616 }
617 } else {
618 // Note the reverse syntax in the assembler.
619 // This cmp will compute srcReg1 - srcReg2
620 if (src1Type.subtypeOf(Type::Bool)) {
621 m_as. cmpb (Reg8(int(srcReg2)), Reg8(int(srcReg1)));
622 } else {
623 m_as. cmp_reg64_reg64(srcReg2, srcReg1);
624 }
625 }
626 }
627
628 emitJccDirectExit(inst, cc);
629 }
630
631 void CodeGenerator::cgJmpGt (IRInstruction* inst) { cgJcc(inst); }
632 void CodeGenerator::cgJmpGte (IRInstruction* inst) { cgJcc(inst); }
633 void CodeGenerator::cgJmpLt (IRInstruction* inst) { cgJcc(inst); }
634 void CodeGenerator::cgJmpLte (IRInstruction* inst) { cgJcc(inst); }
635 void CodeGenerator::cgJmpEq (IRInstruction* inst) { cgJcc(inst); }
636 void CodeGenerator::cgJmpNeq (IRInstruction* inst) { cgJcc(inst); }
637 void CodeGenerator::cgJmpSame (IRInstruction* inst) { cgJcc(inst); }
638 void CodeGenerator::cgJmpNSame(IRInstruction* inst) { cgJcc(inst); }
639
640 /**
641 * Once the arg sources and dests are all assigned; emit moves and exchanges
642 * to put all the args in desired registers. In addition to moves and
643 * exchanges, shuffleArgs also handles adding lea-offsets for dest registers
644 * (dest = src + lea-offset) and zero extending bools (dest = zeroExtend(src)).
645 */
646 typedef Transl::X64Assembler Asm;
647 static void shuffleArgs(Asm& a, ArgGroup& args) {
648 // First schedule arg moves
649 for (size_t i = 0; i < args.size(); ++i) {
650 // We don't support memory-to-register moves currently.
651 assert(args[i].getKind() == ArgDesc::Reg ||
652 args[i].getKind() == ArgDesc::TypeReg ||
653 args[i].getKind() == ArgDesc::Imm ||
654 args[i].getKind() == ArgDesc::Addr ||
655 args[i].getKind() == ArgDesc::None);
656 }
657 // Handle register-to-register moves.
658 int moves[kNumX64Regs];
659 ArgDesc* argDescs[kNumX64Regs];
660 memset(moves, -1, sizeof moves);
661 memset(argDescs, 0, sizeof argDescs);
662 for (size_t i = 0; i < args.size(); ++i) {
663 auto kind = args[i].getKind();
664 if (!(kind == ArgDesc::Reg ||
665 kind == ArgDesc::Addr ||
666 kind == ArgDesc::TypeReg)) {
667 continue;
668 }
669 auto dstReg = args[i].getDstReg();
670 auto srcReg = args[i].getSrcReg();
671 if (dstReg != srcReg) {
672 moves[int(dstReg)] = int(srcReg);
673 argDescs[int(dstReg)] = &args[i];
674 }
675 }
676 std::vector<MoveInfo> howTo;
677 doRegMoves(moves, int(reg::rScratch), howTo);
678 for (size_t i = 0; i < howTo.size(); ++i) {
679 if (howTo[i].m_kind == MoveInfo::Move) {
680 if (howTo[i].m_reg2 == reg::rScratch) {
681 a. movq (howTo[i].m_reg1, howTo[i].m_reg2);
682 } else {
683 ArgDesc* argDesc = argDescs[int(howTo[i].m_reg2)];
684 ArgDesc::Kind kind = argDesc->getKind();
685 if (kind == ArgDesc::Reg || kind == ArgDesc::TypeReg) {
686 if (argDesc->isZeroExtend()) {
687 a. movzbl (rbyte(howTo[i].m_reg1), r32(howTo[i].m_reg2));
688 } else {
689 a. movq (howTo[i].m_reg1, howTo[i].m_reg2);
690 }
691 } else {
692 assert(kind == ArgDesc::Addr);
693 a. lea (howTo[i].m_reg1[argDesc->getImm().q()],
694 howTo[i].m_reg2);
695 }
696 if (kind != ArgDesc::TypeReg) {
697 argDesc->markDone();
698 }
699 }
700 } else {
701 a. xchgq (howTo[i].m_reg1, howTo[i].m_reg2);
702 }
703 }
704 // Handle const-to-register moves, type shifting,
705 // load-effective address and zero extending for bools.
706 // Ignore args that have been handled by the
707 // move above.
708 for (size_t i = 0; i < args.size(); ++i) {
709 if (!args[i].done()) {
710 ArgDesc::Kind kind = args[i].getKind();
711 PhysReg dst = args[i].getDstReg();
712 if (kind == ArgDesc::Imm) {
713 emitLoadImm(a, args[i].getImm().q(), dst);
714 } else if (kind == ArgDesc::TypeReg) {
715 a. shlq (kTypeShiftBits, dst);
716 } else if (kind == ArgDesc::Addr) {
717 a. addq (args[i].getImm(), dst);
718 } else if (args[i].isZeroExtend()) {
719 a. movzbl (rbyte(dst), r32(dst));
720 } else if (RuntimeOption::EvalHHIRGenerateAsserts &&
721 kind == ArgDesc::None) {
722 emitLoadImm(a, 0xbadbadbadbadbad, dst);
723 }
724 }
725 }
726 }
727
728 void CodeGenerator::cgCallNative(Asm& a, IRInstruction* inst) {
729 using namespace NativeCalls;
730 Opcode opc = inst->op();
731 always_assert(CallMap::hasInfo(opc));
732
733 const CallInfo& info = CallMap::getInfo(opc);
734 ArgGroup argGroup;
735 for (auto const& arg : info.args) {
736 SSATmp* src = inst->getSrc(arg.srcIdx);
737 switch (arg.type) {
738 case SSA:
739 argGroup.ssa(src);
740 break;
741 case TV:
742 argGroup.typedValue(src);
743 break;
744 case VecKeyS:
745 argGroup.vectorKeyS(src);
746 break;
747 case VecKeyIS:
748 argGroup.vectorKeyIS(src);
749 break;
750 case Immed:
751 always_assert(0 && "We can't generate a native call for this");
752 break;
753 }
754 }
755
756 TCA addr = nullptr;
757 switch (info.func.type) {
758 case FPtr:
759 addr = info.func.ptr;
760 break;
761 case FSSA:
762 addr = inst->getSrc(info.func.srcIdx)->getValTCA();
763 break;
764 }
765 cgCallHelper(a,
766 addr,
767 info.dest != DestType::None ? inst->getDst(0) : nullptr,
768 info.sync,
769 argGroup,
770 info.dest);
771 }
772
773 void CodeGenerator::cgCallHelper(Asm& a,
774 TCA addr,
775 SSATmp* dst,
776 SyncOptions sync,
777 ArgGroup& args,
778 DestType destType) {
779 PhysReg dstReg0 = InvalidReg;
780 PhysReg dstReg1 = InvalidReg;
781 if (dst) {
782 dstReg0 = dst->getReg(0);
783 dstReg1 = dst->getReg(1);
784 }
785 return cgCallHelper(a, Transl::Call(addr), dstReg0, dstReg1, sync, args,
786 destType);
787 }
788
789 void CodeGenerator::cgCallHelper(Asm& a,
790 TCA addr,
791 PhysReg dstReg,
792 SyncOptions sync,
793 ArgGroup& args,
794 DestType destType) {
795 cgCallHelper(a, Transl::Call(addr), dstReg, InvalidReg, sync, args, destType);
796 }
797
798 void CodeGenerator::cgCallHelper(Asm& a,
799 const Transl::Call& call,
800 PhysReg dstReg,
801 SyncOptions sync,
802 ArgGroup& args,
803 DestType destType) {
804 cgCallHelper(a, call, dstReg, InvalidReg, sync, args, destType);
805 }
806
807 void CodeGenerator::cgCallHelper(Asm& a,
808 const Transl::Call& call,
809 PhysReg dstReg0,
810 PhysReg dstReg1,
811 SyncOptions sync,
812 ArgGroup& args,
813 DestType destType) {
814 cgCallHelper(a, call, dstReg0, dstReg1, sync, args,
815 m_state.liveRegs[m_curInst], destType);
816 }
817
818 void CodeGenerator::cgCallHelper(Asm& a,
819 const Transl::Call& call,
820 PhysReg dstReg0,
821 PhysReg dstReg1,
822 SyncOptions sync,
823 ArgGroup& args,
824 RegSet toSave,
825 DestType destType) {
826 assert(int(args.size()) <= kNumRegisterArgs);
827 assert(m_curInst->isNative());
828
829 // Save the register that are live at the point after this IR instruction.
830 // However, don't save the destination registers that will be overwritten
831 // by this call.
832 toSave = toSave & kCallerSaved;
833 assert((toSave & RegSet().add(dstReg0).add(dstReg1)).empty());
834 PhysRegSaverParity<1> regSaver(a, toSave);
835
836 // Assign registers to the arguments
837 for (size_t i = 0; i < args.size(); i++) {
838 args[i].setDstReg(argNumToRegName[i]);
839 }
840
841 shuffleArgs(a, args);
842
843 // do the call; may use a trampoline
844 m_tx64->emitCall(a, call);
845
846 if (sync != kNoSyncPoint) {
847 recordSyncPoint(a, sync);
848 }
849
850 // copy the call result to the destination register(s)
851 if (destType == DestType::TV) {
852 // rax contains m_type and m_aux but we're expecting just the
853 // type in the lower bits, so shift the type result register.
854 auto rval = packed_tv ? reg::rdx : reg::rax;
855 auto rtyp = packed_tv ? reg::rax : reg::rdx;
856 if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, rtyp);
857 shuffle2(a, rval, rtyp, dstReg0, dstReg1);
858 } else if (destType == DestType::SSA) {
859 // copy the single-register result to dstReg0
860 assert(dstReg1 == InvalidReg);
861 if (dstReg0 != InvalidReg) emitMovRegReg(a, reg::rax, dstReg0);
862 } else {
863 // void return type, no registers have values
864 assert(dstReg0 == InvalidReg && dstReg1 == InvalidReg);
865 }
866 }
867
868 void CodeGenerator::cgMov(IRInstruction* inst) {
869 assert(!inst->getSrc(0)->hasReg(1)); // TODO: t2082361: handle Gen & Cell
870 SSATmp* dst = inst->getDst();
871 SSATmp* src = inst->getSrc(0);
872 auto dstReg = dst->getReg();
873 if (!src->hasReg(0)) {
874 assert(src->isConst());
875 if (src->type() == Type::Bool) {
876 emitLoadImm(m_as, (int64_t)src->getValBool(), dstReg);
877 } else {
878 emitLoadImm(m_as, src->getValRawInt(), dstReg);
879 }
880 } else {
881 auto srcReg = src->getReg();
882 emitMovRegReg(m_as, srcReg, dstReg);
883 }
884 }
885
886 template<class OpInstr, class Oper>
887 void CodeGenerator::cgUnaryIntOp(SSATmp* dst,
888 SSATmp* src,
889 OpInstr instr,
890 Oper oper) {
891 if (src->type() != Type::Int && src->type() != Type::Bool) {
892 assert(0); CG_PUNT(UnaryIntOp);
893 }
894 auto dstReg = dst->getReg();
895 auto srcReg = src->getReg();
896 assert(dstReg != InvalidReg);
897 auto& a = m_as;
898
899 // Integer operations require 64-bit representations
900 zeroExtendIfBool(a, src);
901
902 if (srcReg != InvalidReg) {
903 emitMovRegReg(a, srcReg, dstReg);
904 (a.*instr) (dstReg);
905 } else {
906 assert(src->isConst());
907 emitLoadImm(a, oper(src->getValRawInt()), dstReg);
908 }
909 }
910
911 void CodeGenerator::cgNotWork(SSATmp* dst, SSATmp* src) {
912 cgUnaryIntOp(dst, src, &Asm::not, [](int64_t i) { return ~i; });
913 }
914
915 void CodeGenerator::cgNegateWork(SSATmp* dst, SSATmp* src) {
916 cgUnaryIntOp(dst, src, &Asm::neg, [](int64_t i) { return -i; });
917 }
918
919 void CodeGenerator::cgNegate(IRInstruction* inst) {
920 cgNegateWork(inst->getDst(), inst->getSrc(0));
921 }
922
923 inline static Reg8 convertToReg8(PhysReg reg) { return rbyte(reg); }
924 inline static Reg64 convertToReg64(PhysReg reg) { return reg; }
925
926 template<class Oper, class RegType>
927 void CodeGenerator::cgBinaryIntOp(IRInstruction* inst,
928 void (Asm::*instrIR)(Immed, RegType),
929 void (Asm::*instrRR)(RegType, RegType),
930 void (Asm::*movInstr)(RegType, RegType),
931 void (*extendInstr)(Asm&, const SSATmp*),
932 Oper oper,
933 RegType (*convertReg)(PhysReg),
934 Commutativity commuteFlag) {
935 const SSATmp* dst = inst->getDst();
936 const SSATmp* src1 = inst->getSrc(0);
937 const SSATmp* src2 = inst->getSrc(1);
938 if (!(src1->isA(Type::Bool) || src1->isA(Type::Int)) ||
939 !(src2->isA(Type::Bool) || src2->isA(Type::Int))) {
940 CG_PUNT(cgBinaryIntOp);
941 }
942
943 bool const commutative = commuteFlag == Commutative;
944 auto const dstReg = dst->getReg();
945 auto const src1Reg = src1->getReg();
946 auto const src2Reg = src2->getReg();
947 auto& a = m_as;
948 bool const useBoolOps = src1->isA(Type::Bool) && src2->isA(Type::Bool);
949
950 auto const dstOpReg = convertReg(dstReg);
951 auto const src1OpReg = convertReg(src1Reg);
952 auto const src2OpReg = convertReg(src2Reg);
953 auto const rOpScratch = convertReg(rScratch);
954
955 // We can only use byte operations if both operands are boolean. If so, we'll
956 // sign extend *after* the operation is complete to save a movzbl.
957 if (!useBoolOps) {
958 // Extend booleans: integer operations require 64-bit
959 // representations.
960 zeroExtendIfBool(m_as, src1);
961 zeroExtendIfBool(m_as, src2);
962 }
963
964 auto signExtendIfNecessary = [&] {
965 if (useBoolOps) {
966 extendInstr(m_as, dst);
967 }
968 };
969
970 // Two registers.
971 if (src1Reg != InvalidReg && src2Reg != InvalidReg) {
972 if (dstReg == src1Reg) {
973 (a.*instrRR) (src2OpReg, dstOpReg);
974 } else if (dstReg == src2Reg) {
975 if (commutative) {
976 (a.*instrRR) (src1OpReg, dstOpReg);
977 } else {
978 (a.*movInstr)(src1OpReg, rOpScratch);
979 (a.*instrRR) (src2OpReg, rOpScratch);
980 (a.*movInstr)(rOpScratch, dstOpReg);
981 }
982 } else {
983 emitMovRegReg(a, src1Reg, dstReg);
984 (a.*instrRR) (src2OpReg, dstOpReg);
985 }
986 signExtendIfNecessary();
987 return;
988 }
989
990 // Two immediates.
991 if (src1Reg == InvalidReg && src2Reg == InvalidReg) {
992 assert(src1->isConst() && src2->isConst());
993 int64_t value = useBoolOps ?
994 (int64_t)oper(src1->getValBool(), src2->getValBool()) :
995 oper(src1->getValRawInt(), src2->getValRawInt());
996 emitLoadImm(a, value, dstReg);
997 return;
998 }
999
1000 // One register, and one immediate.
1001 if (commutative) {
1002 auto immedSrc = (src2Reg == InvalidReg ? src2 : src1);
1003 auto immed = useBoolOps ?
1004 (int64_t)immedSrc->getValBool() : immedSrc->getValRawInt();
1005 auto srcReg = (src2Reg == InvalidReg ? src1 : src2)->getReg();
1006 if (srcReg == dstReg) {
1007 (a.*instrIR) (immed, dstOpReg);
1008 } else {
1009 emitLoadImm(a, immed, dstReg);
1010 (a.*instrRR) (convertReg(srcReg), dstOpReg);
1011 }
1012 signExtendIfNecessary();
1013 return;
1014 }
1015
1016 // NonCommutative:
1017 if (src1Reg == InvalidReg) {
1018 if (dstReg == src2Reg) {
1019 emitLoadImm(a, useBoolOps ?
1020 (int64_t)src1->getValBool() : src1->getValRawInt(), rScratch);
1021 (a.*instrRR) (src2OpReg, rOpScratch);
1022 (a.*movInstr)(rOpScratch, dstOpReg);
1023 } else {
1024 emitLoadImm(a, useBoolOps ?
1025 (int64_t)src1->getValBool() : src1->getValRawInt(), dstReg);
1026 (a.*instrRR) (src2OpReg, dstOpReg);
1027 }
1028 signExtendIfNecessary();
1029 return;
1030 }
1031
1032 assert(src2Reg == InvalidReg);
1033 emitMovRegReg(a, src1Reg, dstReg);
1034 (a.*instrIR) (
1035 useBoolOps ? src2->getValBool() : src2->getValRawInt(), dstOpReg);
1036 signExtendIfNecessary();
1037 }
1038
1039 template<class Oper, class RegType>
1040 void CodeGenerator::cgBinaryOp(IRInstruction* inst,
1041 void (Asm::*instrIR)(Immed, RegType),
1042 void (Asm::*instrRR)(RegType, RegType),
1043 void (Asm::*movInstr)(RegType, RegType),
1044 void (Asm::*fpInstr)(RegXMM, RegXMM),
1045 void (*extendInstr)(Asm&, const SSATmp*),
1046 Oper oper,
1047 RegType (*convertReg)(PhysReg),
1048 Commutativity commuteFlag) {
1049 const SSATmp* dst = inst->getDst();
1050 const SSATmp* src1 = inst->getSrc(0);
1051 const SSATmp* src2 = inst->getSrc(1);
1052 if (!(src1->isA(Type::Bool) || src1->isA(Type::Int) || src1->isA(Type::Dbl))
1053 ||
1054 !(src2->isA(Type::Bool) || src2->isA(Type::Int) || src2->isA(Type::Dbl)) )
1055 {
1056 CG_PUNT(cgBinaryOp);
1057 }
1058 if (src1->isA(Type::Dbl) || src2->isA(Type::Dbl)) {
1059 prepBinaryXmmOp(m_as, src1, src2);
1060 (m_as.*fpInstr)(xmm1, xmm0);
1061 m_as. mov_xmm_reg64(xmm0, dst->getReg());
1062 return;
1063 }
1064 cgBinaryIntOp(inst, instrIR, instrRR, movInstr, extendInstr,
1065 oper, convertReg, commuteFlag);
1066 }
1067
1068 bool CodeGenerator::emitIncDecHelper(SSATmp* dst, SSATmp* src1, SSATmp* src2,
1069 void(Asm::*emitFunc)(Reg64)) {
1070 if (src1->getReg() != InvalidReg &&
1071 dst ->getReg() != InvalidReg &&
1072 src1->isA(Type::Int) &&
1073 // src2 == 1:
1074 src2->isConst() && src2->isA(Type::Int) && src2->getValInt() == 1) {
1075 emitMovRegReg(m_as, src1->getReg(), dst->getReg());
1076 (m_as.*emitFunc)(dst->getReg());
1077 return true;
1078 }
1079 return false;
1080 }
1081
1082 /*
1083 * If src2 is 1, this generates dst = src1 + 1 using the "inc" x86 instruction.
1084 * The return value is whether or not the instruction could be generated.
1085 */
1086 bool CodeGenerator::emitInc(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
1087 return emitIncDecHelper(dst, src1, src2, &Asm::incq);
1088 }
1089
1090 /*
1091 * If src2 is 1, this generates dst = src1 - 1 using the "dec" x86 instruction.
1092 * The return value is whether or not the instruction could be generated.
1093 */
1094 bool CodeGenerator::emitDec(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
1095 return emitIncDecHelper(dst, src1, src2, &Asm::decq);
1096 }
1097
1098 void CodeGenerator::cgOpAdd(IRInstruction* inst) {
1099 SSATmp* dst = inst->getDst();
1100 SSATmp* src1 = inst->getSrc(0);
1101 SSATmp* src2 = inst->getSrc(1);
1102
1103 // Special cases: x = y + 1
1104 if (emitInc(dst, src1, src2) || emitInc(dst, src2, src1)) return;
1105
1106 if (src1->isA(Type::Bool) & src2->isA(Type::Bool))
1107 {
1108 cgBinaryIntOp(inst,
1109 &Asm::addb,
1110 &Asm::addb,
1111 &Asm::movb,
1112 &zeroExtendIfBool,
1113 std::plus<bool>(),
1114 &convertToReg8,
1115 Commutative);
1116 } else {
1117 cgBinaryOp(inst,
1118 &Asm::addq,
1119 &Asm::addq,
1120 &Asm::movq,
1121 &Asm::addsd_xmm_xmm,
1122 nullptr, // not used
1123 std::plus<int64_t>(),
1124 &convertToReg64,
1125 Commutative);
1126 }
1127 }
1128
1129 void CodeGenerator::cgOpSub(IRInstruction* inst) {
1130 SSATmp* dst = inst->getDst();
1131 SSATmp* src1 = inst->getSrc(0);
1132 SSATmp* src2 = inst->getSrc(1);
1133
1134 if (emitDec(dst, src1, src2)) return;
1135
1136 if (src1->isConst() && src1->isA(Type::Int) && src1->getValInt() == 0 &&
1137 !src2->isA(Type::Dbl)) {
1138 return cgNegateWork(dst, src2);
1139 }
1140
1141 if (src1->isA(Type::Bool) & src2->isA(Type::Bool)) {
1142 cgBinaryIntOp(inst,
1143 &Asm::subb,
1144 &Asm::subb,
1145 &Asm::movb,
1146 &signExtendBool, // bool "-1" needs to be sign extended as int
1147 std::minus<bool>(),
1148 &convertToReg8,
1149 NonCommutative);
1150 } else {
1151 cgBinaryOp(inst,
1152 &Asm::subq,
1153 &Asm::subq,
1154 &Asm::movq,
1155 &Asm::subsd_xmm_xmm,
1156 nullptr, // not used
1157 std::minus<int64_t>(),
1158 &convertToReg64,
1159 NonCommutative);
1160 }
1161 }
1162
1163 void CodeGenerator::cgOpAnd(IRInstruction* inst) {
1164 SSATmp* src1 = inst->getSrc(0);
1165 SSATmp* src2 = inst->getSrc(1);
1166
1167 if (src1->isA(Type::Bool) & src2->isA(Type::Bool)) {
1168 cgBinaryIntOp(inst,
1169 &Asm::andb,
1170 &Asm::andb,
1171 &Asm::movb,
1172 &zeroExtendIfBool,
1173 [] (bool a, bool b) { return a & b; },
1174 &convertToReg8,
1175 Commutative);
1176 } else {
1177 cgBinaryIntOp(inst,
1178 &Asm::andq,
1179 &Asm::andq,
1180 &Asm::movq,
1181 nullptr, // not used
1182 [] (int64_t a, int64_t b) { return a & b; },
1183 &convertToReg64,
1184 Commutative);
1185 }
1186 }
1187
1188 void CodeGenerator::cgOpOr(IRInstruction* inst) {
1189 SSATmp* src1 = inst->getSrc(0);
1190 SSATmp* src2 = inst->getSrc(1);
1191
1192 if (src1->isA(Type::Bool) & src2->isA(Type::Bool)) {
1193 cgBinaryIntOp(inst,
1194 &Asm::orb,
1195 &Asm::orb,
1196 &Asm::movb,
1197 &zeroExtendIfBool,
1198 [] (bool a, bool b) { return a | b; },
1199 &convertToReg8,
1200 Commutative);
1201 } else {
1202 cgBinaryIntOp(inst,
1203 &Asm::orq,
1204 &Asm::orq,
1205 &Asm::movq,
1206 nullptr, // not used
1207 [] (int64_t a, int64_t b) { return a | b; },
1208 &convertToReg64,
1209 Commutative);
1210 }
1211 }
1212
1213 void CodeGenerator::cgOpDiv(IRInstruction* inst) {
1214 not_implemented();
1215 }
1216
1217 void CodeGenerator::cgOpXor(IRInstruction* inst) {
1218 SSATmp* dst = inst->getDst();
1219 SSATmp* src1 = inst->getSrc(0);
1220 SSATmp* src2 = inst->getSrc(1);
1221 if (src2->isConst() && src2->type() == Type::Int &&
1222 src2->getValInt() == ~0L) {
1223 return cgNotWork(dst, src1);
1224 }
1225
1226 if (src1->isA(Type::Bool) & src2->isA(Type::Bool)) {
1227 cgBinaryIntOp(inst,
1228 &Asm::xorb,
1229 &Asm::xorb,
1230 &Asm::movb,
1231 &zeroExtendIfBool,
1232 [] (bool a, bool b) { return a ^ b; },
1233 &convertToReg8,
1234 Commutative);
1235 } else {
1236 cgBinaryIntOp(inst,
1237 &Asm::xorq,
1238 &Asm::xorq,
1239 &Asm::movq,
1240 nullptr, // not used
1241 [] (int64_t a, int64_t b) { return a ^ b; },
1242 &convertToReg64,
1243 Commutative);
1244 }
1245 }
1246
1247 void CodeGenerator::cgOpMul(IRInstruction* inst) {
1248 SSATmp* src1 = inst->getSrc(0);
1249 SSATmp* src2 = inst->getSrc(1);
1250
1251 if (src1->isA(Type::Bool) & src2->isA(Type::Bool)) {
1252 cgBinaryIntOp(inst,
1253 &Asm::andb,
1254 &Asm::andb,
1255 &Asm::movb,
1256 &zeroExtendIfBool,
1257 [] (bool a, bool b) { return a & b; },
1258 &convertToReg8,
1259 Commutative);
1260 } else {
1261 // Boolean multiplication is the same as &
1262 cgBinaryOp(inst,
1263 &Asm::imul,
1264 &Asm::imul,
1265 &Asm::movq,
1266 &Asm::mulsd_xmm_xmm,
1267 nullptr, // not used
1268 std::multiplies<int64_t>(),
1269 &convertToReg64,
1270 Commutative);
1271 }
1272 }
1273
1274 void CodeGenerator::cgOpNot(IRInstruction* inst) {
1275 auto const src = inst->getSrc(0);
1276 auto const dstReg = inst->getDst()->getReg();
1277 auto& a = m_as;
1278
1279 if (src->isConst()) {
1280 a. movb (!src->getValBool(), rbyte(dstReg));
1281 } else {
1282 if (dstReg != src->getReg()) {
1283 a. movb (rbyte(src->getReg()), rbyte(dstReg));
1284 }
1285 a. xorb (1, rbyte(dstReg));
1286 }
1287 }
1288
1289 ///////////////////////////////////////////////////////////////////////////////
1290 // Comparison Operators
1291 ///////////////////////////////////////////////////////////////////////////////
1292
1293 #define DISPATCHER(name) \
1294 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, StringData* a2) \
1295 { return name(a1, a2); } \
1296 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, int64_t a2) \
1297 { return name(a1, a2); } \
1298 HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, ObjectData* a2) \
1299 { return name(a1, Object(a2)); } \
1300 HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, ObjectData* a2) \
1301 { return name(Object(a1), Object(a2)); } \
1302 HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, int64_t a2) \
1303 { return name(Object(a1), a2); } \
1304 HOT_FUNC_VM int64_t ccmp_ ## name (ArrayData* a1, ArrayData* a2) \
1305 { return name(Array(a1), Array(a2)); }
1306
1307 DISPATCHER(same)
1308 DISPATCHER(equal)
1309 DISPATCHER(more)
1310 DISPATCHER(less)
1311
1312 #undef DISPATCHER
1313
1314 template <typename A, typename B>
1315 inline int64_t ccmp_nsame(A a, B b) { return !ccmp_same(a, b); }
1316
1317 template <typename A, typename B>
1318 inline int64_t ccmp_nequal(A a, B b) { return !ccmp_equal(a, b); }
1319
1320 template <typename A, typename B>
1321 inline int64_t ccmp_lte(A a, B b) { return !ccmp_more(a, b); }
1322
1323 template <typename A, typename B>
1324 inline int64_t ccmp_gte(A a, B b) { return !ccmp_less(a, b); }
1325
1326 #define CG_OP_CMP(inst, setter, name) \
1327 cgOpCmpHelper(inst, &Asm:: setter, ccmp_ ## name, ccmp_ ## name, \
1328 ccmp_ ## name, ccmp_ ## name, ccmp_ ## name, ccmp_ ## name)
1329
1330 // SRON - string, resource, object, or number
1331 static bool typeIsSRON(Type t) {
1332 return t.isString()
1333 || t == Type::Obj // encompases object and resource
1334 || t == Type::Int
1335 || t == Type::Dbl
1336 ;
1337 }
1338
1339 void CodeGenerator::cgOpCmpHelper(
1340 IRInstruction* inst,
1341 void (Asm::*setter)(Reg8),
1342 int64_t (*str_cmp_str)(StringData*, StringData*),
1343 int64_t (*str_cmp_int)(StringData*, int64_t),
1344 int64_t (*str_cmp_obj)(StringData*, ObjectData*),
1345 int64_t (*obj_cmp_obj)(ObjectData*, ObjectData*),
1346 int64_t (*obj_cmp_int)(ObjectData*, int64_t),
1347 int64_t (*arr_cmp_arr)( ArrayData*, ArrayData*)
1348 ) {
1349 SSATmp* dst = inst->getDst();
1350 SSATmp* src1 = inst->getSrc(0);
1351 SSATmp* src2 = inst->getSrc(1);
1352
1353 Type type1 = src1->type();
1354 Type type2 = src2->type();
1355
1356 auto src1Reg = src1->getReg();
1357 auto src2Reg = src2->getReg();
1358 auto dstReg = dst ->getReg();
1359
1360 auto setFromFlags = [&] {
1361 (m_as.*setter)(rbyte(dstReg));
1362 };
1363 // It is possible that some pass has been done after simplification; if such
1364 // a pass invalidates our invariants, then just punt.
1365
1366 // simplifyCmp has done const-const optimization
1367 //
1368 // If the types are the same and there is only one constant,
1369 // simplifyCmp has moved it to the right.
1370 if (src1->isConst()) {
1371 CG_PUNT(cgOpCmpHelper_const);
1372 }
1373
1374 /////////////////////////////////////////////////////////////////////////////
1375 // case 1: null/string cmp string
1376 // simplifyCmp has converted the null to ""
1377 if (type1.isString() && type2.isString()) {
1378 ArgGroup args;
1379 args.ssa(src1).ssa(src2);
1380 cgCallHelper(m_as, (TCA)str_cmp_str, dst, kSyncPoint, args);
1381 }
1382
1383 /////////////////////////////////////////////////////////////////////////////
1384 // case 2: bool/null cmp anything
1385 // simplifyCmp has converted all args to bool
1386 else if (type1 == Type::Bool && type2 == Type::Bool) {
1387 if (src2->isConst()) {
1388 m_as. cmpb (src2->getValBool(), Reg8(int(src1Reg)));
1389 } else {
1390 m_as. cmpb (Reg8(int(src2Reg)), Reg8(int(src1Reg)));
1391 }
1392 setFromFlags();
1393 }
1394
1395 /////////////////////////////////////////////////////////////////////////////
1396 // case 3, 4, and 7: string/resource/object/number (sron) cmp sron
1397 // These cases must be amalgamated because Type::Obj can refer to an object
1398 // or to a resource.
1399 // strings are canonicalized to the left, ints to the right
1400 else if (typeIsSRON(type1) && typeIsSRON(type2)) {
1401 // the common case: int cmp int
1402 if (type1 == Type::Int && type2 == Type::Int) {
1403 if (src2->isConst()) {
1404 m_as.cmp_imm64_reg64(src2->getValInt(), src1Reg);
1405 } else {
1406 m_as.cmp_reg64_reg64(src2Reg, src1Reg);
1407 }
1408 setFromFlags();
1409 }
1410
1411 else if (type1 == Type::Dbl || type2 == Type::Dbl) {
1412 if ((type1 == Type::Dbl || type1 == Type::Int) &&
1413 (type2 == Type::Dbl || type2 == Type::Int)) {
1414 prepBinaryXmmOp(m_as, src1, src2);
1415 doubleCmp(m_as, xmm0, xmm1);
1416 setFromFlags();
1417 } else {
1418 CG_PUNT(cgOpCmpHelper_Dbl);
1419 }
1420 }
1421
1422 else if (type1.isString()) {
1423 // string cmp string is dealt with in case 1
1424 // string cmp double is punted above
1425
1426 if (type2 == Type::Int) {
1427 ArgGroup args;
1428 args.ssa(src1).ssa(src2);
1429 cgCallHelper(m_as, (TCA)str_cmp_int, dst, kSyncPoint, args);
1430 } else if (type2 == Type::Obj) {
1431 ArgGroup args;
1432 args.ssa(src1).ssa(src2);
1433 cgCallHelper(m_as, (TCA)str_cmp_obj, dst, kSyncPoint, args);
1434 } else {
1435 CG_PUNT(cgOpCmpHelper_sx);
1436 }
1437 }
1438
1439 else if (type1 == Type::Obj) {
1440 // string cmp object/resource is dealt with above
1441 // object cmp double is punted above
1442
1443 if (type2 == Type::Obj) {
1444 ArgGroup args;
1445 args.ssa(src1).ssa(src2);
1446 cgCallHelper(m_as, (TCA)obj_cmp_obj, dst, kSyncPoint, args);
1447 } else if (type2 == Type::Int) {
1448 ArgGroup args;
1449 args.ssa(src1).ssa(src2);
1450 cgCallHelper(m_as, (TCA)obj_cmp_int, dst, kSyncPoint, args);
1451 } else {
1452 CG_PUNT(cgOpCmpHelper_ox);
1453 }
1454 }
1455
1456 else NOT_REACHED();
1457 }
1458
1459 /////////////////////////////////////////////////////////////////////////////
1460 // case 5: array cmp array
1461 else if (type1.isArray() && type2.isArray()) {
1462 ArgGroup args;
1463 args.ssa(src1).ssa(src2);
1464 cgCallHelper(m_as, (TCA)arr_cmp_arr, dst, kSyncPoint, args);
1465 }
1466
1467 /////////////////////////////////////////////////////////////////////////////
1468 // case 6: array cmp anything
1469 // simplifyCmp has already dealt with this case.
1470
1471 /////////////////////////////////////////////////////////////////////////////
1472 else {
1473 // We have a type which is not a common type. It might be a cell or a box.
1474 CG_PUNT(cgOpCmpHelper_unimplemented);
1475 }
1476 }
1477
1478 void CodeGenerator::cgOpEq(IRInstruction* inst) {
1479 CG_OP_CMP(inst, sete, equal);
1480 }
1481
1482 void CodeGenerator::cgOpNeq(IRInstruction* inst) {
1483 CG_OP_CMP(inst, setne, nequal);
1484 }
1485
1486 void CodeGenerator::cgOpSame(IRInstruction* inst) {
1487 CG_OP_CMP(inst, sete, same);
1488 }
1489
1490 void CodeGenerator::cgOpNSame(IRInstruction* inst) {
1491 CG_OP_CMP(inst, setne, nsame);
1492 }
1493
1494 void CodeGenerator::cgOpLt(IRInstruction* inst) {
1495 CG_OP_CMP(inst, setl, less);
1496 }
1497
1498 void CodeGenerator::cgOpGt(IRInstruction* inst) {
1499 CG_OP_CMP(inst, setg, more);
1500 }
1501
1502 void CodeGenerator::cgOpLte(IRInstruction* inst) {
1503 CG_OP_CMP(inst, setle, lte);
1504 }
1505
1506 void CodeGenerator::cgOpGte(IRInstruction* inst) {
1507 CG_OP_CMP(inst, setge, gte);
1508 }
1509
1510 ///////////////////////////////////////////////////////////////////////////////
1511 // Type check operators
1512 ///////////////////////////////////////////////////////////////////////////////
1513
1514 template<class OpndType>
1515 ConditionCode CodeGenerator::emitTypeTest(Type type, OpndType src,
1516 bool negate) {
1517 assert(!type.subtypeOf(Type::Cls));
1518 ConditionCode cc;
1519 if (type.isString()) {
1520 emitTestTVType(m_as, KindOfStringBit, src);
1521 cc = CC_NZ;
1522 } else if (type.equals(Type::UncountedInit)) {
1523 emitTestTVType(m_as, KindOfUncountedInitBit, src);
1524 cc = CC_NZ;
1525 } else if (type.equals(Type::Uncounted)) {
1526 emitCmpTVType(m_as, KindOfRefCountThreshold, src);
1527 cc = CC_LE;
1528 } else if (type.equals(Type::Cell)) {
1529 emitCmpTVType(m_as, KindOfRef, src);
1530 cc = CC_L;
1531 } else if (type.equals(Type::Gen)) {
1532 return CC_None; // nothing to check
1533 } else {
1534 DataType dataType = type.toDataType();
1535 assert(dataType == KindOfRef ||
1536 (dataType >= KindOfUninit && dataType <= KindOfObject));
1537 emitCmpTVType(m_as, dataType, src);
1538 cc = CC_E;
1539 }
1540 return negate ? ccNegate(cc) : cc;
1541 }
1542
1543 template<class OpndType>
1544 ConditionCode CodeGenerator::emitTypeTest(IRInstruction* inst, OpndType src,
1545 bool negate) {
1546 return emitTypeTest(inst->getTypeParam(), src, negate);
1547 }
1548
1549 void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) {
1550 if (cc == CC_None) return;
1551 m_as.setcc(cc, rbyte(inst->getDst()->getReg()));
1552 }
1553
1554 ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) {
1555 if (inst->getTypeParam().subtypeOf(Type::Obj)) {
1556 // Task #2094715: Handle isType tests for Type::Obj, which
1557 // requires checking that checked object is not a resource
1558 CG_PUNT(cgIsTypeObject);
1559 }
1560
1561 SSATmp* src = inst->getSrc(0);
1562 if (src->isA(Type::PtrToGen)) {
1563 PhysReg base = src->getReg();
1564 return emitTypeTest(inst, base[TVOFF(m_type)], negate);
1565 }
1566 assert(src->isA(Type::Gen));
1567 assert(!src->isConst());
1568 PhysReg srcReg = src->getReg(1); // type register
1569 return emitTypeTest(inst, srcReg, negate);
1570 }
1571
1572 template<class OpndType>
1573 void CodeGenerator::emitGuardType(OpndType src, IRInstruction* inst) {
1574 emitGuardOrFwdJcc(inst, emitTypeTest(inst, src, true));
1575 }
1576
1577 void CodeGenerator::cgGuardTypeCell(PhysReg baseReg,
1578 int64_t offset,
1579 IRInstruction* inst) {
1580 emitGuardType(baseReg[offset + TVOFF(m_type)], inst);
1581 }
1582
1583 void CodeGenerator::cgIsTypeMemCommon(IRInstruction* inst, bool negate) {
1584 emitSetCc(inst, emitIsTypeTest(inst, negate));
1585 }
1586
1587 void CodeGenerator::cgIsTypeCommon(IRInstruction* inst, bool negate) {
1588 emitSetCc(inst, emitIsTypeTest(inst, negate));
1589 }
1590
1591 void CodeGenerator::cgJmpIsTypeCommon(IRInstruction* inst, bool negate) {
1592 emitJccDirectExit(inst, emitIsTypeTest(inst, negate));
1593 }
1594
1595 void CodeGenerator::cgIsType(IRInstruction* inst) {
1596 cgIsTypeCommon(inst, false);
1597 }
1598
1599 void CodeGenerator::cgIsNType(IRInstruction* inst) {
1600 cgIsTypeCommon(inst, true);
1601 }
1602
1603 void CodeGenerator::cgJmpIsType(IRInstruction* inst) {
1604 cgJmpIsTypeCommon(inst, false);
1605 }
1606
1607 void CodeGenerator::cgJmpIsNType(IRInstruction* inst) {
1608 cgJmpIsTypeCommon(inst, true);
1609 }
1610
1611 void CodeGenerator::cgIsTypeMem(IRInstruction* inst) {
1612 cgIsTypeMemCommon(inst, false);
1613 }
1614
1615 void CodeGenerator::cgIsNTypeMem(IRInstruction* inst) {
1616 cgIsTypeMemCommon(inst, true);
1617 }
1618
1619 ///////////////////////////////////////////////////////////////////////////////
1620
1621 HOT_FUNC_VM static bool instanceOfHelper(const Class* objClass,
1622 const Class* testClass) {
1623 return testClass && objClass->classof(testClass);
1624 }
1625
1626 HOT_FUNC_VM static bool instanceOfHelperIFace(const Class* objClass,
1627 const Class* testClass) {
1628 return testClass && objClass->classof(testClass->preClass());
1629 }
1630
1631 void CodeGenerator::cgInstanceOf(IRInstruction* inst) {
1632 const bool ifaceHint = inst->getSrc(2)->getValBool();
1633 cgCallHelper(m_as,
1634 TCA(ifaceHint ? instanceOfHelperIFace : instanceOfHelper),
1635 inst->getDst(),
1636 kNoSyncPoint,
1637 ArgGroup()
1638 .ssa(inst->getSrc(0))
1639 .ssa(inst->getSrc(1)));
1640 }
1641
1642 /*
1643 * Check instanceof using instance bitmasks.
1644 *
1645 * Note it's not necessary to check whether the test class is defined:
1646 * if it doesn't exist than the candidate can't be an instance of it
1647 * and will fail this check.
1648 */
1649 void CodeGenerator::emitInstanceBitmaskCheck(IRInstruction* inst) {
1650 auto const rObjClass = inst->getSrc(0)->getReg(0);
1651 auto const testClassName = inst->getSrc(1)->getValStr();
1652 auto& a = m_as;
1653
1654 int offset;
1655 uint8_t mask;
1656 if (!Class::getInstanceBitMask(testClassName, offset, mask)) {
1657 always_assert(!"cgInstanceOfBitmask had no bitmask");
1658 }
1659 a. testb (int8_t(mask), rObjClass[offset]);
1660 }
1661
1662 void CodeGenerator::cgInstanceOfBitmask(IRInstruction* inst) {
1663 auto& a = m_as;
1664 emitInstanceBitmaskCheck(inst);
1665 a. setnz (rbyte(inst->getDst()->getReg()));
1666 }
1667
1668 void CodeGenerator::cgNInstanceOfBitmask(IRInstruction* inst) {
1669 auto& a = m_as;
1670 emitInstanceBitmaskCheck(inst);
1671 a. setz (rbyte(inst->getDst()->getReg()));
1672 }
1673
1674 void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction* inst) {
1675 emitInstanceBitmaskCheck(inst);
1676 emitJccDirectExit(inst, CC_NZ);
1677 }
1678
1679 void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction* inst) {
1680 emitInstanceBitmaskCheck(inst);
1681 emitJccDirectExit(inst, CC_Z);
1682 }
1683
1684 /*
1685 * Check instanceof using the superclass vector on the end of the
1686 * Class entry.
1687 */
1688 void CodeGenerator::cgExtendsClass(IRInstruction* inst) {
1689 auto const rObjClass = inst->getSrc(0)->getReg();
1690 auto const testClass = inst->getSrc(1)->getValClass();
1691 auto rTestClass = inst->getSrc(1)->getReg();
1692 auto const rdst = rbyte(inst->getDst()->getReg());
1693 auto& a = m_as;
1694
1695 Label out;
1696 Label notExact;
1697 Label falseLabel;
1698
1699 if (rTestClass == InvalidReg) { // TODO(#2031606)
1700 rTestClass = rScratch; // careful below about asm-x64 smashing this
1701 emitLoadImm(a, (int64_t)testClass, rTestClass);
1702 }
1703
1704 // Test if it is the exact same class. TODO(#2044801): we should be
1705 // doing this control flow at the IR level.
1706 if (!(testClass->attrs() & AttrAbstract)) {
1707 a. cmpq (rTestClass, rObjClass);
1708 a. jne8 (notExact);
1709 a. movb (1, rdst);
1710 a. jmp8 (out);
1711 }
1712
1713 auto const vecOffset = Class::classVecOff() +
1714 sizeof(Class*) * (testClass->classVecLen() - 1);
1715
1716 // Check the length of the class vectors---if the candidate's is at
1717 // least as long as the potential base (testClass) it might be a
1718 // subclass.
1719 asm_label(a, notExact);
1720 a. cmpl (testClass->classVecLen(),
1721 rObjClass[Class::classVecLenOff()]);
1722 a. jb8 (falseLabel);
1723
1724 // If it's a subclass, rTestClass must be at the appropriate index.
1725 a. cmpq (rTestClass, rObjClass[vecOffset]);
1726 a. sete (rdst);
1727 a. jmp8 (out);
1728
1729 asm_label(a, falseLabel);
1730 a. xorl (r32(rdst), r32(rdst));
1731
1732 asm_label(a, out);
1733 }
1734
1735 void CodeGenerator::cgConvDblToBool(IRInstruction* inst) {
1736 SSATmp* dst = inst->getDst();
1737 auto dstReg = dst->getReg();
1738 assert(dstReg != InvalidReg);
1739 SSATmp* src = inst->getSrc(0);
1740 auto srcReg = src->getReg();
1741 if (srcReg == InvalidReg) {
1742 assert(src->isConst());
1743 double constVal = src->getValDbl();
1744 if (constVal == 0.0) {
1745 m_as.xor_reg64_reg64(dstReg, dstReg);
1746 } else {
1747 m_as.mov_imm64_reg(1, dstReg);
1748 }
1749 } else {
1750 m_as.movq(srcReg, dstReg);
1751 m_as.shlq(1, dstReg); // 0.0 stays zero and -0.0 is now 0.0
1752 m_as.setne(rbyte(dstReg)); // lower byte becomes 1 if dstReg != 0
1753 m_as.movzbl(rbyte(dstReg), r32(dstReg));
1754 }
1755 }
1756
1757 void CodeGenerator::cgConvIntToBool(IRInstruction* inst) {
1758 SSATmp* dst = inst->getDst();
1759 auto dstReg = dst->getReg();
1760 assert(dstReg != InvalidReg);
1761 SSATmp* src = inst->getSrc(0);
1762 auto srcReg = src->getReg();
1763
1764 if (srcReg == InvalidReg) {
1765 assert(src->isConst());
1766 int64_t constVal = src->getValInt();
1767 if (constVal == 0) {
1768 m_as.xor_reg64_reg64(dstReg, dstReg);
1769 } else {
1770 m_as.mov_imm64_reg(1, dstReg);
1771 }
1772 } else {
1773 m_as.test_reg64_reg64(srcReg, srcReg);
1774 m_as.setne(rbyte(dstReg));
1775 m_as.movzbl(rbyte(dstReg), r32(dstReg));
1776 }
1777 }
1778
1779 void CodeGenerator::cgConvBoolToDbl(IRInstruction* inst) {
1780 // cvtsi2sd doesn't modify the high bits of its target, which can
1781 // cause false dependencies to prevent register renaming from kicking
1782 // in. Break the dependency chain by zeroing out xmm0.
1783 m_as.pxor_xmm_xmm(xmm0, xmm0);
1784 SSATmp* dst = inst->getDst();
1785 auto dstReg = dst->getReg();
1786 assert(dstReg != InvalidReg);
1787 SSATmp* src = inst->getSrc(0);
1788 auto srcReg = src->getReg();
1789 if (srcReg == InvalidReg) {
1790 assert(src->isConst());
1791 int64_t constVal = src->getValRawInt();
1792 if (constVal == 0) {
1793 m_as.xor_reg64_reg64(dstReg, dstReg);
1794 } else {
1795 m_as.mov_imm64_reg(1, dstReg);
1796 }
1797 } else {
1798 m_as.movzbl(rbyte(srcReg), r32(dstReg));
1799 }
1800 m_as.cvtsi2sd_reg64_xmm(dstReg, xmm0);
1801 m_as.mov_xmm_reg64(xmm0, dstReg);
1802 }
1803
1804 void CodeGenerator::cgConvIntToDbl(IRInstruction* inst) {
1805 // cvtsi2sd doesn't modify the high bits of its target, which can
1806 // cause false dependencies to prevent register renaming from kicking
1807 // in. Break the dependency chain by zeroing out xmm0.
1808 m_as.pxor_xmm_xmm(xmm0, xmm0);
1809 SSATmp* dst = inst->getDst();
1810 auto dstReg = dst->getReg();
1811 assert(dstReg != InvalidReg);
1812 SSATmp* src = inst->getSrc(0);
1813 auto srcReg = src->getReg();
1814 if (srcReg == InvalidReg) {
1815 assert(src->isConst());
1816 int64_t constVal = src->getValRawInt();
1817 if (constVal == 0) {
1818 m_as.xor_reg64_reg64(dstReg, dstReg);
1819 } else {
1820 m_as.mov_imm64_reg(constVal, dstReg);
1821 }
1822 m_as.cvtsi2sd_reg64_xmm(dstReg, xmm0);
1823 } else {
1824 m_as.cvtsi2sd_reg64_xmm(srcReg, xmm0);
1825 }
1826 m_as.mov_xmm_reg64(xmm0, dstReg);
1827 }
1828
1829 void CodeGenerator::cgConvBoolToInt(IRInstruction* inst) {
1830 SSATmp* dst = inst->getDst();
1831 auto dstReg = dst->getReg();
1832 assert(dstReg != InvalidReg);
1833 SSATmp* src = inst->getSrc(0);
1834 auto srcReg = src->getReg();
1835 assert(src->isConst() == (srcReg == InvalidReg));
1836 if (srcReg == InvalidReg) {
1837 int64_t constVal = src->getValRawInt();
1838 if (constVal == 0) {
1839 m_as.xor_reg64_reg64(dstReg, dstReg);
1840 } else {
1841 m_as.mov_imm64_reg(1, dstReg);
1842 }
1843 } else {
1844 m_as.movzbl(rbyte(srcReg), r32(dstReg));
1845 }
1846 }
1847
1848 void CodeGenerator::cgConvBoolToStr(IRInstruction* inst) {
1849 SSATmp* dst = inst->getDst();
1850 auto dstReg = dst->getReg();
1851 assert(dstReg != InvalidReg);
1852 SSATmp* src = inst->getSrc(0);
1853 auto srcReg = src->getReg();
1854 assert(src->isConst() == (srcReg == InvalidReg));
1855 if (srcReg == InvalidReg) {
1856 auto constVal = src->getValBool();
1857 if (!constVal) {
1858 m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
1859 } else {
1860 m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), dstReg);
1861 }
1862 } else {
1863 m_as.testb(Reg8(int(srcReg)), Reg8(int(srcReg)));
1864 m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
1865 m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), rScratch);
1866 m_as.cmov_reg64_reg64(CC_NZ, rScratch, dstReg);
1867 }
1868 }
1869
1870 void CodeGenerator::cgUnboxPtr(IRInstruction* inst) {
1871 SSATmp* dst = inst->getDst();
1872 SSATmp* src = inst->getSrc(0);
1873
1874 auto srcReg = src->getReg();
1875 auto dstReg = dst->getReg();
1876
1877 assert(srcReg != InvalidReg);
1878 assert(dstReg != InvalidReg);
1879
1880 emitMovRegReg(m_as, srcReg, dstReg);
1881 emitDerefIfVariant(m_as, PhysReg(dstReg));
1882 }
1883
1884 void CodeGenerator::cgUnbox(IRInstruction* inst) {
1885 SSATmp* dst = inst->getDst();
1886 SSATmp* src = inst->getSrc(0);
1887 auto dstValReg = dst->getReg(0);
1888 auto dstTypeReg = dst->getReg(1);
1889 auto srcValReg = src->getReg(0);
1890 auto srcTypeReg = src->getReg(1);
1891
1892 assert(dstValReg != dstTypeReg);
1893 assert(src->type().equals(Type::Gen));
1894 assert(dst->type().notBoxed());
1895
1896 emitCmpTVType(m_as, HPHP::KindOfRef, srcTypeReg);
1897 ifThenElse(CC_E, [&] {
1898 // srcTypeReg == KindOfRef; srcValReg is RefData*
1899 const size_t ref_tv_off = RefData::tvOffset();
1900 if (dstValReg != srcValReg) {
1901 m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
1902 emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
1903 r32(dstTypeReg));
1904 } else {
1905 emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
1906 r32(dstTypeReg));
1907 m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
1908 }
1909 }, [&] {
1910 // srcTypeReg != KindOfRef; copy src -> dst
1911 shuffle2(m_as, srcValReg, srcTypeReg, dstValReg, dstTypeReg);
1912 });
1913 }
1914
1915 void CodeGenerator::cgLdFuncCachedCommon(IRInstruction* inst) {
1916 SSATmp* dst = inst->getDst();
1917 SSATmp* methodName = inst->getSrc(0);
1918
1919 const StringData* name = methodName->getValStr();
1920 CacheHandle ch = TargetCache::allocFixedFunction(name);
1921 size_t funcCacheOff = ch + offsetof(FixedFuncCache, m_func);
1922
1923 auto dstReg = dst->getReg();
1924 if (dstReg == InvalidReg) {
1925 // happens if LdFixedFunc and FCall not in same trace
1926 m_as. cmpq(0, rVmTl[funcCacheOff]);
1927 } else {
1928 m_as. loadq (rVmTl[funcCacheOff], dstReg);
1929 m_as. testq (dstReg, dstReg);
1930 }
1931 }
1932
1933 void CodeGenerator::cgLdFuncCached(IRInstruction* inst) {
1934 cgLdFuncCachedCommon(inst);
1935 // jz off to the helper call in astubs
1936 unlikelyIfBlock(CC_Z, [&] (Asm& a) {
1937 // this helper tries the autoload map, and fatals on failure
1938 cgCallNative(a, inst);
1939 });
1940 }
1941
1942
1943 void CodeGenerator::cgLdFuncCachedSafe(IRInstruction* inst) {
1944 cgLdFuncCachedCommon(inst);
1945 if (Block* taken = inst->getTaken()) {
1946 emitFwdJcc(m_as, CC_Z, taken);
1947 }
1948 }
1949
1950 void CodeGenerator::cgLdFunc(IRInstruction* inst) {
1951 SSATmp* dst = inst->getDst();
1952 SSATmp* methodName = inst->getSrc(0);
1953
1954 TargetCache::CacheHandle ch = TargetCache::FuncCache::alloc();
1955 // raises an error if function not found
1956 cgCallHelper(m_as, (TCA)FuncCache::lookup, dst->getReg(), kSyncPoint,
1957 ArgGroup().imm(ch).ssa(methodName));
1958 }
1959
1960 static void emitLdObjClass(CodeGenerator::Asm& a,
1961 PhysReg objReg,
1962 PhysReg dstReg) {
1963 a.loadq (objReg[ObjectData::getVMClassOffset()], dstReg);
1964 }
1965
1966 void CodeGenerator::cgLdObjClass(IRInstruction* inst) {
1967 auto dstReg = inst->getDst()->getReg();
1968 auto objReg = inst->getSrc(0)->getReg();
1969
1970 emitLdObjClass(m_as, objReg, dstReg);
1971 }
1972
1973 void CodeGenerator::cgLdObjMethod(IRInstruction *inst) {
1974 auto cls = inst->getSrc(0);
1975 auto clsReg = cls->getReg();
1976 auto name = inst->getSrc(1);
1977 auto actRec = inst->getSrc(2);
1978 auto actRecReg = actRec->getReg();
1979 CacheHandle handle = Transl::TargetCache::MethodCache::alloc();
1980
1981 // lookup in the targetcache
1982 assert(MethodCache::kNumLines == 1);
1983 if (debug) {
1984 MethodCache::Pair p;
1985 static_assert(sizeof(p.m_value) == 8,
1986 "MethodCache::Pair::m_value assumed to be 8 bytes");
1987 static_assert(sizeof(p.m_key) == 8,
1988 "MethodCache::Pair::m_key assumed to be 8 bytes");
1989 }
1990
1991 // preload handle->m_value
1992 m_as.loadq(rVmTl[handle + offsetof(MethodCache::Pair, m_value)], rScratch);
1993 m_as.cmpq (rVmTl[handle + offsetof(MethodCache::Pair, m_key)], clsReg);
1994 ifThenElse(CC_E, // if handle->key == cls
1995 [&] { // then actReg->m_func = handle->value
1996 m_as.storeq(rScratch, actRecReg[AROFF(m_func)]);
1997 },
1998 [&] { // else call slow path helper
1999 cgCallHelper(m_as, (TCA)methodCacheSlowPath, InvalidReg,
2000 kSyncPoint,
2001 ArgGroup().addr(rVmTl, handle)
2002 .ssa(actRec)
2003 .ssa(name)
2004 .ssa(cls));
2005 });
2006 }
2007
2008 void CodeGenerator::cgRetVal(IRInstruction* inst) {
2009 auto const rFp = inst->getSrc(0)->getReg();
2010 auto* const val = inst->getSrc(1);
2011 auto& a = m_as;
2012
2013 // Store return value at the top of the caller's eval stack
2014 // (a) Store the type
2015 if (val->type().needsReg()) {
2016 emitStoreTVType(a, val->getReg(1), rFp[AROFF(m_r) + TVOFF(m_type)]);
2017 } else {
2018 emitStoreTVType(a, val->type().toDataType(),
2019 rFp[AROFF(m_r) + TVOFF(m_type)]);
2020 }
2021
2022 // (b) Store the actual value (not necessary when storing Null)
2023 if (val->type().isNull()) return;
2024 if (val->inst()->op() == DefConst) {
2025 a. storeq (val->getValRawInt(),
2026 rFp[AROFF(m_r) + TVOFF(m_data)]);
2027 } else {
2028 zeroExtendIfBool(m_as, val);
2029 a. storeq (val->getReg(), rFp[AROFF(m_r) + TVOFF(m_data)]);
2030 }
2031 }
2032
2033 void CodeGenerator::cgRetAdjustStack(IRInstruction* inst) {
2034 auto const rFp = inst->getSrc(0)->getReg();
2035 auto const dstSp = inst->getDst()->getReg();
2036 auto& a = m_as;
2037 a. lea (rFp[AROFF(m_r)], dstSp);
2038 }
2039
2040 void CodeGenerator::cgLdRetAddr(IRInstruction* inst) {
2041 auto fpReg = inst->getSrc(0)->getReg(0);
2042 assert(fpReg != InvalidReg);
2043 m_as.push(fpReg[AROFF(m_savedRip)]);
2044 }
2045
2046 void checkFrame(ActRec* fp, Cell* sp, bool checkLocals) {
2047 const Func* func = fp->m_func;
2048 if (fp->hasVarEnv()) {
2049 assert(fp->getVarEnv()->getCfp() == fp);
2050 }
2051 // TODO: validate this pointer from actrec
2052 int numLocals = func->numLocals();
2053 DEBUG_ONLY Cell* firstSp = ((Cell*)fp) - func->numSlotsInFrame();
2054 assert(sp <= firstSp || func->isGenerator());
2055 if (checkLocals) {
2056 int numParams = func->numParams();
2057 for (int i=0; i < numLocals; i++) {
2058 if (i >= numParams && func->isGenerator() && i < func->numNamedLocals()) {
2059 continue;
2060 }
2061 assert(checkTv(frame_local(fp, i)));
2062 }
2063 }
2064 // We unfortunately can't do the same kind of check for the stack
2065 // because it may contain ActRecs.
2066 #if 0
2067 for (Cell* c=sp; c < firstSp; c++) {
2068 TypedValue* tv = (TypedValue*)c;
2069 assert(tvIsPlausible(tv));
2070 DataType t = tv->m_type;
2071 if (IS_REFCOUNTED_TYPE(t)) {
2072 assert(tv->m_data.pstr->getCount() > 0);
2073 }
2074 }
2075 #endif
2076 }
2077
2078 void traceRet(ActRec* fp, Cell* sp, void* rip) {
2079 if (rip == TranslatorX64::Get()->getCallToExit()) {
2080 return;
2081 }
2082 checkFrame(fp, sp, /*checkLocals*/ false);
2083 assert(sp <= (Cell*)fp || fp->m_func->isGenerator());
2084 // check return value if stack not empty
2085 if (sp < (Cell*)fp) assertTv(sp);
2086 }
2087
2088 void CodeGenerator::emitTraceRet(CodeGenerator::Asm& a) {
2089 // call to a trace function
2090 a. movq (rVmFp, rdi);
2091 a. movq (rVmSp, rsi);
2092 a. loadq (*rsp, rdx); // return ip from native stack
2093 // do the call; may use a trampoline
2094 m_tx64->emitCall(a, TCA(traceRet));
2095 }
2096
2097 void CodeGenerator::cgRetCtrl(IRInstruction* inst) {
2098 SSATmp* sp = inst->getSrc(0);
2099 SSATmp* fp = inst->getSrc(1);
2100
2101 // Make sure rVmFp and rVmSp are set appropriately
2102 emitMovRegReg(m_as, sp->getReg(), rVmSp);
2103 emitMovRegReg(m_as, fp->getReg(), rVmFp);
2104
2105 // Return control to caller
2106 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2107 emitTraceRet(m_as);
2108 }
2109 m_as.ret();
2110 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2111 m_as.ud2();
2112 }
2113 }
2114
2115 void CodeGenerator::emitReqBindAddr(const Func* func,
2116 TCA& dest,
2117 Offset offset) {
2118 dest = m_tx64->emitServiceReq(TranslatorX64::SRFlags::SRNone,
2119 REQ_BIND_ADDR,
2120 2ull,
2121 &dest,
2122 offset);
2123 }
2124
2125 void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) {
2126 JmpSwitchData* data = inst->getExtra<JmpSwitchDest>();
2127 SSATmp* index = inst->getSrc(0);
2128 auto indexReg = index->getReg();
2129
2130 if (!index->isConst()) {
2131 if (data->bounded) {
2132 if (data->base) {
2133 m_as. subq(data->base, indexReg);
2134 }
2135 m_as. cmpq(data->cases - 2, indexReg);
2136 m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen);
2137 TCA def = m_tx64->emitServiceReq(REQ_BIND_JMPCC_SECOND, 3,
2138 m_as.code.frontier, data->defaultOff, CC_AE);
2139 m_as. jae(def);
2140 }
2141
2142 TCA* table = m_tx64->m_globalData.alloc<TCA>(sizeof(TCA), data->cases);
2143 TCA afterLea = m_as.code.frontier + TranslatorX64::kLeaRipLen;
2144 ptrdiff_t diff = (TCA)table - afterLea;
2145 assert(deltaFits(diff, sz::dword));
2146 m_as. lea(rip[diff], rScratch);
2147 assert(m_as.code.frontier == afterLea);
2148 m_as. jmp(rScratch[indexReg*8]);
2149
2150 for (int i = 0; i < data->cases; i++) {
2151 emitReqBindAddr(data->func, table[i], data->targets[i]);
2152 }
2153 } else {
2154 int64_t indexVal = index->getValInt();
2155
2156 if (data->bounded) {
2157 indexVal -= data->base;
2158 if (indexVal >= data->cases - 2 || indexVal < 0) {
2159 m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->defaultOff));
2160 return;
2161 }
2162 }
2163 m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->targets[indexVal]));
2164 }
2165 }
2166
2167 typedef FixedStringMap<TCA,true> SSwitchMap;
2168
2169 static TCA sswitchHelperFast(const StringData* val,
2170 const SSwitchMap* table,
2171 TCA* def) {
2172 TCA* dest = table->find(val);
2173 return dest ? *dest : *def;
2174 }
2175
2176 void CodeGenerator::cgLdSSwitchDestFast(IRInstruction* inst) {
2177 auto data = inst->getExtra<LdSSwitchDestFast>();
2178
2179 auto table = m_tx64->m_globalData.alloc<SSwitchMap>(64);
2180 table->init(data->numCases);
2181 for (int64_t i = 0; i < data->numCases; ++i) {
2182 table->add(data->cases[i].str, nullptr);
2183 TCA* addr = table->find(data->cases[i].str);
2184 emitReqBindAddr(data->func, *addr, data->cases[i].dest);
2185 }
2186 TCA* def = m_tx64->m_globalData.alloc<TCA>(sizeof(TCA), 1);
2187 emitReqBindAddr(data->func, *def, data->defaultOff);
2188
2189 cgCallHelper(m_as,
2190 TCA(sswitchHelperFast),
2191 inst->getDst(),
2192 kNoSyncPoint,
2193 ArgGroup()
2194 .ssa(inst->getSrc(0))
2195 .immPtr(table)
2196 .immPtr(def));
2197 }
2198
2199 static TCA sswitchHelperSlow(TypedValue typedVal,
2200 const StringData** strs,
2201 int numStrs,
2202 TCA* jmptab) {
2203 TypedValue* cell = tvToCell(&typedVal);
2204 for (int i = 0; i < numStrs; ++i) {
2205 if (tvAsCVarRef(cell).equal(strs[i])) return jmptab[i];
2206 }
2207 return jmptab[numStrs]; // default case
2208 }
2209
2210 void CodeGenerator::cgLdSSwitchDestSlow(IRInstruction* inst) {
2211 auto data = inst->getExtra<LdSSwitchDestSlow>();
2212
2213 auto strtab = m_tx64->m_globalData.alloc<const StringData*>(
2214 sizeof(const StringData*), data->numCases);
2215 auto jmptab = m_tx64->m_globalData.alloc<TCA>(sizeof(TCA),
2216 data->numCases + 1);
2217 for (int i = 0; i < data->numCases; ++i) {
2218 strtab[i] = data->cases[i].str;
2219 emitReqBindAddr(data->func, jmptab[i], data->cases[i].dest);
2220 }
2221 emitReqBindAddr(data->func, jmptab[data->numCases], data->defaultOff);
2222
2223 cgCallHelper(m_as,
2224 TCA(sswitchHelperSlow),
2225 inst->getDst(),
2226 kSyncPoint,
2227 ArgGroup()
2228 .typedValue(inst->getSrc(0))
2229 .immPtr(strtab)
2230 .imm(data->numCases)
2231 .immPtr(jmptab));
2232 }
2233
2234 /*
2235 * It'd be nice not to have the cgMov here (and just copy propagate
2236 * the source or something), but for now we're keeping it allocated to
2237 * rVmFp so inlined calls to C++ helpers that use the rbp chain to
2238 * find the caller's ActRec will work correctly.
2239 *
2240 * This instruction primarily exists to assist in optimizing away
2241 * unused activation records, so it's usually not going to happen
2242 * anyway.
2243 */
2244 void CodeGenerator::cgDefInlineFP(IRInstruction* inst) {
2245 auto const fp = inst->getSrc(0)->getReg();
2246 auto const fakeRet = m_tx64->getRetFromInlinedFrame();
2247 auto const retBCOff = inst->getExtra<DefInlineFP>()->offset;
2248
2249 m_as. storeq (fakeRet, fp[AROFF(m_savedRip)]);
2250 m_as. storel (retBCOff, fp[AROFF(m_soff)]);
2251
2252 cgMov(inst);
2253 }
2254
2255 void CodeGenerator::cgInlineReturn(IRInstruction* inst) {
2256 auto fpReg = inst->getSrc(0)->getReg();
2257 assert(fpReg == rVmFp);
2258 m_as. loadq (fpReg[AROFF(m_savedRbp)], rVmFp);
2259 }
2260
2261 void CodeGenerator::cgReDefSP(IRInstruction* inst) {
2262 // TODO(#2288359): this instruction won't be necessary (for
2263 // non-generator frames) when we don't track rVmSp independently
2264 // from rVmFp. In generator frames we'll have to track offsets from
2265 // a DefGeneratorSP or something similar.
2266 auto fp = inst->getSrc(0)->getReg();
2267 auto dst = inst->getDst()->getReg();
2268 auto off = -inst->getExtra<ReDefSP>()->offset * sizeof(Cell);
2269 emitLea(m_as, fp[off], dst);
2270 }
2271
2272 void CodeGenerator::cgStashGeneratorSP(IRInstruction* inst) {
2273 cgMov(inst);
2274 }
2275
2276 void CodeGenerator::cgReDefGeneratorSP(IRInstruction* inst) {
2277 cgMov(inst);
2278 }
2279
2280 void CodeGenerator::cgFreeActRec(IRInstruction* inst) {
2281 m_as.loadq(inst->getSrc(0)->getReg()[AROFF(m_savedRbp)],
2282 inst->getDst()->getReg());
2283 }
2284
2285 void CodeGenerator::cgAllocSpill(IRInstruction* inst) {
2286 SSATmp* numSlots = inst->getSrc(0);
2287
2288 assert(numSlots->isConst());
2289 int64_t n = numSlots->getValInt();
2290 assert(n >= 0 && n % 2 == 0);
2291 if (n > 0) {
2292 m_as.sub_imm32_reg64(spillSlotsToSize(n), reg::rsp);
2293 }
2294 }
2295
2296 void CodeGenerator::cgFreeSpill(IRInstruction* inst) {
2297 SSATmp* numSlots = inst->getSrc(0);
2298
2299 assert(numSlots->isConst());
2300 int64_t n = numSlots->getValInt();
2301 assert(n >= 0 && n % 2 == 0);
2302 if (n > 0) {
2303 m_as.add_imm32_reg64(spillSlotsToSize(n), reg::rsp);
2304 }
2305 }
2306
2307 void CodeGenerator::cgSpill(IRInstruction* inst) {
2308 SSATmp* dst = inst->getDst();
2309 SSATmp* src = inst->getSrc(0);
2310
2311 assert(dst->numNeededRegs() == src->numNeededRegs());
2312 for (int locIndex = 0; locIndex < src->numNeededRegs(); ++locIndex) {
2313 auto srcReg = src->getReg(locIndex);
2314
2315 // We do not need to mask booleans, since the IR will reload the spill
2316 auto sinfo = dst->getSpillInfo(locIndex);
2317 assert(sinfo.type() == SpillInfo::Memory);
2318 m_as. storeq(srcReg, reg::rsp[sizeof(uint64_t) * sinfo.mem()]);
2319 }
2320 }
2321
2322 void CodeGenerator::cgReload(IRInstruction* inst) {
2323 SSATmp* dst = inst->getDst();
2324 SSATmp* src = inst->getSrc(0);
2325
2326 assert(dst->numNeededRegs() == src->numNeededRegs());
2327 for (int locIndex = 0; locIndex < src->numNeededRegs(); ++locIndex) {
2328 auto dstReg = dst->getReg(locIndex);
2329
2330 auto sinfo = src->getSpillInfo(locIndex);
2331 assert(sinfo.type() == SpillInfo::Memory);
2332 m_as. loadq(reg::rsp[sizeof(uint64_t) * sinfo.mem()], dstReg);
2333 }
2334 }
2335
2336 void CodeGenerator::cgStPropWork(IRInstruction* inst, bool genTypeStore) {
2337 SSATmp* obj = inst->getSrc(0);
2338 SSATmp* prop = inst->getSrc(1);
2339 SSATmp* src = inst->getSrc(2);
2340 cgStore(obj->getReg(), prop->getValInt(), src, genTypeStore);
2341 }
2342 void CodeGenerator::cgStProp(IRInstruction* inst) {
2343 cgStPropWork(inst, true);
2344 }
2345 void CodeGenerator::cgStPropNT(IRInstruction* inst) {
2346 cgStPropWork(inst, false);
2347 }
2348
2349 void CodeGenerator::cgStMemWork(IRInstruction* inst, bool genStoreType) {
2350 SSATmp* addr = inst->getSrc(0);
2351 SSATmp* offset = inst->getSrc(1);
2352 SSATmp* src = inst->getSrc(2);
2353 cgStore(addr->getReg(), offset->getValInt(), src, genStoreType);
2354 }
2355 void CodeGenerator::cgStMem(IRInstruction* inst) {
2356 cgStMemWork(inst, true);
2357 }
2358 void CodeGenerator::cgStMemNT(IRInstruction* inst) {
2359 cgStMemWork(inst, false);
2360 }
2361
2362 void CodeGenerator::cgStRefWork(IRInstruction* inst, bool genStoreType) {
2363 auto destReg = inst->getDst()->getReg();
2364 auto addrReg = inst->getSrc(0)->getReg();
2365 SSATmp* src = inst->getSrc(1);
2366 cgStore(addrReg, RefData::tvOffset(), src, genStoreType);
2367 if (destReg != InvalidReg) emitMovRegReg(m_as, addrReg, destReg);
2368 }
2369
2370 void CodeGenerator::cgStRef(IRInstruction* inst) {
2371 cgStRefWork(inst, true);
2372 }
2373 void CodeGenerator::cgStRefNT(IRInstruction* inst) {
2374 cgStRefWork(inst, false);
2375 }
2376
2377 static int64_t getLocalOffset(int64_t index) {
2378 return -cellsToBytes(index + 1);
2379 }
2380
2381 static int64_t getLocalOffset(SSATmp* index) {
2382 return getLocalOffset(index->getValInt());
2383 }
2384
2385 int CodeGenerator::getIterOffset(SSATmp* tmp) {
2386 const Func* func = getCurFunc();
2387 int64_t index = tmp->getValInt();
2388 return -cellsToBytes(((index + 1) * kNumIterCells + func->numLocals()));
2389 }
2390
2391 void CodeGenerator::cgStLoc(IRInstruction* inst) {
2392 cgStore(inst->getSrc(0)->getReg(),
2393 getLocalOffset(inst->getExtra<StLoc>()->locId),
2394 inst->getSrc(1),
2395 true /* store type */);
2396 }
2397
2398 void CodeGenerator::cgStLocNT(IRInstruction* inst) {
2399 cgStore(inst->getSrc(0)->getReg(),
2400 getLocalOffset(inst->getExtra<StLocNT>()->locId),
2401 inst->getSrc(1),
2402 false /* store type */);
2403 }
2404
2405 void CodeGenerator::cgSyncVMRegs(IRInstruction* inst) {
2406 emitMovRegReg(m_as, inst->getSrc(0)->getReg(), rVmFp);
2407 emitMovRegReg(m_as, inst->getSrc(1)->getReg(), rVmSp);
2408 }
2409
2410 void CodeGenerator::cgExitTrace(IRInstruction* inst) {
2411 SSATmp* func = inst->getSrc(0);
2412 SSATmp* pc = inst->getSrc(1);
2413 SSATmp* sp = inst->getSrc(2);
2414 SSATmp* fp = inst->getSrc(3);
2415 SSATmp* notTakenPC = nullptr;
2416 IRInstruction* toSmash = nullptr;
2417 assert(pc->isConst() && inst->getNumSrcs() <= 6);
2418
2419 TraceExitType::ExitType exitType = getExitType(inst->op());
2420 if (exitType == TraceExitType::Normal && inst->getExtra<ExitTrace>()) {
2421 // Unconditional trace exit
2422 toSmash = inst->getExtra<ExitTrace>()->toSmash;
2423 assert(toSmash);
2424 } else if (exitType == TraceExitType::NormalCc) {
2425 // Exit at trace end which is the target of a conditional branch
2426 notTakenPC = inst->getSrc(4);
2427 assert(notTakenPC->isConst());
2428 if (inst->getExtra<ExitTraceCc>()) {
2429 toSmash = inst->getExtra<ExitTraceCc>()->toSmash;
2430 assert(toSmash);
2431 }
2432 }
2433 using namespace HPHP::VM::Transl;
2434
2435 Asm& a = m_as; // Note: m_as is the same as m_atubs for Exit Traces,
2436 // unless exit trace was moved to end of main trace
2437
2438 emitMovRegReg(a, sp->getReg(), rVmSp);
2439 emitMovRegReg(a, fp->getReg(), rVmFp);
2440
2441 // Get the SrcKey for the dest
2442 SrcKey destSK(func->getValFunc(), pc->getValInt());
2443
2444 switch (exitType) {
2445 case TraceExitType::NormalCc:
2446 if (toSmash) {
2447 TCA smashAddr = toSmash->getTCA();
2448 if (smashAddr == kIRDirectJmpInactive) {
2449 // The jump in the main trace has been optimized away
2450 // this exit trace is no longer needed
2451 break;
2452 }
2453 // Patch the original jcc;jmp, don't emit another
2454 IRInstruction* jcc = toSmash;
2455 Opcode opc = jcc->op();
2456 ConditionCode cc = queryJmpToCC(opc);
2457 uint64_t taken = pc->getValInt();
2458 uint64_t notTaken = notTakenPC->getValInt();
2459
2460 m_astubs.setcc(cc, rbyte(serviceReqArgRegs[4]));
2461 m_tx64->emitServiceReq(TranslatorX64::SRFlags::SRInline,
2462 REQ_BIND_JMPCC_FIRST,
2463 4ull,
2464 smashAddr,
2465 taken,
2466 notTaken,
2467 uint64_t(cc));
2468 } else {
2469 // NormalCc exit but not optimized to jcc directly to destination
2470 m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP);
2471 }
2472 break;
2473 case TraceExitType::Normal:
2474 {
2475 TCA smashAddr = toSmash ? toSmash->getTCA() : nullptr;
2476 if (smashAddr) {
2477 assert(smashAddr != kIRDirectJmpInactive);
2478 if (smashAddr != kIRDirectJccJmpActive) {
2479 // kIRDirectJccJmpActive only needs NormalCc exit in astubs
2480
2481 m_tx64->emitServiceReq(TranslatorX64::SRFlags::SRInline,
2482 REQ_BIND_JMP, 2,
2483 smashAddr,
2484 uint64_t(destSK.offset()));
2485
2486 }
2487 } else {
2488 assert(smashAddr == kIRDirectJmpInactive);
2489 m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP);
2490 }
2491 }
2492 break;
2493 case TraceExitType::Slow:
2494 case TraceExitType::SlowNoProgress:
2495 if (RuntimeOption::EnableInstructionCounts ||
2496 HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) {
2497 Stats::emitInc(m_as,
2498 Stats::opcodeToIRPostStatCounter(
2499 Op(*getCurFunc()->unit()->at(destSK.m_offset))),
2500 -1,
2501 Transl::CC_None,
2502 true);
2503 }
2504
2505 if (HPHP::Trace::moduleEnabled(HPHP::Trace::punt, 1)) {
2506 VM::Op op = (VM::Op)*func->getValFunc()->unit()->at(destSK.m_offset);
2507 std::string name = folly::format(
2508 "exitSlow{}-{}",
2509 exitType == TraceExitType::SlowNoProgress ? "-np" : "",
2510 VM::opcodeToName(op)).str();
2511 m_tx64->emitRecordPunt(a, name);
2512 }
2513 if (RuntimeOption::EvalHHIRDisableTx64) {
2514 // Emit a service request to interpret a single instruction before
2515 // creating a new translation
2516 m_tx64->emitServiceReq(TranslatorX64::SRFlags::SRInline,
2517 REQ_INTERPRET,
2518 2ull, uint64_t(destSK.offset()), 1);
2519 } else {
2520 if (exitType == TraceExitType::Slow) {
2521 m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP_NO_IR);
2522 } else { // SlowNoProgress
2523 m_tx64->emitReqRetransNoIR(a, destSK);
2524 }
2525 }
2526 break;
2527
2528 case TraceExitType::GuardFailure: {
2529 SrcRec* destSR = m_tx64->getSrcRec(destSK);
2530 m_tx64->emitFallbackUncondJmp(a, *destSR);
2531 break;
2532 }
2533 }
2534 }
2535
2536 void CodeGenerator::cgExitTraceCc(IRInstruction* inst) {
2537 cgExitTrace(inst);
2538 }
2539
2540 void CodeGenerator::cgExitSlow(IRInstruction* inst) {
2541 cgExitTrace(inst);
2542 }
2543
2544 void CodeGenerator::cgExitSlowNoProgress(IRInstruction* inst) {
2545 cgExitTrace(inst);
2546 }
2547
2548 void CodeGenerator::cgExitGuardFailure(IRInstruction* inst) {
2549 cgExitTrace(inst);
2550 }
2551
2552 static void emitAssertFlagsNonNegative(CodeGenerator::Asm& as) {
2553 ifThen(as, CC_NGE, [&] { as.ud2(); });
2554 }
2555
2556 static void emitAssertRefCount(CodeGenerator::Asm& as, PhysReg base) {
2557 as.cmpl(HPHP::RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
2558 ifThen(as, CC_NBE, [&] { as.ud2(); });
2559 }
2560
2561 static void emitIncRef(CodeGenerator::Asm& as, PhysReg base) {
2562 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2563 emitAssertRefCount(as, base);
2564 }
2565 // emit incref
2566 as.incl(base[FAST_REFCOUNT_OFFSET]);
2567 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2568 // Assert that the ref count is greater than zero
2569 emitAssertFlagsNonNegative(as);
2570 }
2571 }
2572
2573 void CodeGenerator::cgIncRefWork(Type type, SSATmp* src) {
2574 assert(type.maybeCounted());
2575 auto increfMaybeStatic = [&] {
2576 auto base = src->getReg(0);
2577 if (!type.needsStaticBitCheck()) {
2578 emitIncRef(m_as, base);
2579 } else {
2580 m_as.cmpl(RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
2581 ifThen(m_as, CC_NE, [&] { emitIncRef(m_as, base); });
2582 }
2583 };
2584
2585 if (type.isKnownDataType()) {
2586 assert(IS_REFCOUNTED_TYPE(type.toDataType()));
2587 increfMaybeStatic();
2588 } else {
2589 m_as.cmpl(KindOfRefCountThreshold, r32(src->getReg(1)));
2590 ifThen(m_as, CC_NLE, [&] { increfMaybeStatic(); });
2591 }
2592 }
2593
2594 void CodeGenerator::cgIncRef(IRInstruction* inst) {
2595 SSATmp* dst = inst->getDst();
2596 SSATmp* src = inst->getSrc(0);
2597 Type type = src->type();
2598
2599 cgIncRefWork(type, src);
2600 shuffle2(m_as, src->getReg(0), src->getReg(1),
2601 dst->getReg(0), dst->getReg(1));
2602 }
2603
2604 void CodeGenerator::cgDecRefStack(IRInstruction* inst) {
2605 cgDecRefMem(inst->getTypeParam(),
2606 inst->getSrc(0)->getReg(),
2607 cellsToBytes(inst->getExtra<DecRefStack>()->offset),
2608 nullptr);
2609 }
2610
2611 void CodeGenerator::cgDecRefThis(IRInstruction* inst) {
2612 SSATmp* fp = inst->getSrc(0);
2613 Block* exit = inst->getTaken();
2614 auto fpReg = fp->getReg();
2615 auto scratchReg = rScratch;
2616
2617 // Load AR->m_this into rScratch
2618 m_as.loadq(fpReg[AROFF(m_this)], scratchReg);
2619
2620 auto decrefIfAvailable = [&] {
2621 // Check if this is available and we're not in a static context instead
2622 m_as.testb(1, rbyte(scratchReg));
2623 ifThen(m_as, CC_Z, [&] {
2624 // In the case where the refCount hits zero, we need to store zero
2625 // back to m_this in case a local destructor does debug_backtrace.
2626 cgDecRefStaticType(
2627 Type::Obj,
2628 scratchReg,
2629 exit,
2630 true /* genZeroCheck */,
2631 [&] (Asm& a) {
2632 a. storeq (0, fpReg[AROFF(m_this)]);
2633 }
2634 );
2635 });
2636 };
2637
2638 if (getCurFunc()->isPseudoMain()) {
2639 // In pseudo-mains, emit check for presence of m_this
2640 m_as.testq(scratchReg, scratchReg);
2641 ifThen(m_as, CC_NZ, [&] { decrefIfAvailable(); });
2642 } else {
2643 decrefIfAvailable();
2644 }
2645 }
2646
2647 void CodeGenerator::cgDecRefKillThis(IRInstruction* inst) {
2648 auto const src = inst->getSrc(0);
2649 auto const fpReg = inst->getSrc(1)->getReg();
2650
2651 assert(!inst->getTaken());
2652
2653 cgDecRefStaticType(
2654 src->type(),
2655 src->getReg(),
2656 nullptr,
2657 true /* genZeroCheck */,
2658 [&] (Asm& a) {
2659 a. storeq (0, fpReg[AROFF(m_this)]);
2660 }
2661 );
2662 }
2663
2664 void CodeGenerator::cgDecRefLoc(IRInstruction* inst) {
2665 cgDecRefMem(inst->getTypeParam(),
2666 inst->getSrc(0)->getReg(),
2667 getLocalOffset(inst->getExtra<DecRefLoc>()->locId),
2668 inst->getTaken());
2669 }
2670
2671 void CodeGenerator::cgGenericRetDecRefs(IRInstruction* inst) {
2672 auto const rFp = inst->getSrc(0)->getReg();
2673 auto const retVal = inst->getSrc(1);
2674 auto const numLocals = inst->getSrc(2)->getValInt();
2675 auto const rDest = inst->getDst()->getReg();
2676 auto& a = m_as;
2677
2678 RegSet retvalRegs = retVal->getRegs();
2679 assert(retvalRegs.size() <= 2);
2680
2681 /*
2682 * The generic decref helpers preserve these two registers.
2683 *
2684 * XXX/TODO: we ideally shouldn't be moving the return value into
2685 * these regs and then back; it'd be better to precolor allocation
2686 * this way, or failing that remap the return value SSATmp with a
2687 * separate instruction. This scheme also won't work for us once we
2688 * want hackIR to support handling surprise flags, because
2689 * setprofile will look on the VM stack for the return value.
2690 */
2691 auto spillRegs = RegSet(r14).add(r15);
2692
2693 /*
2694 * Since we're making a call using a custom ABI to the generic
2695 * decref helper, it's important that our src and dest registers are
2696 * allocated to the registers we expect, and that no other SSATmp's
2697 * are still allocated to registers at this time.
2698 */
2699 const auto UNUSED expectedLiveRegs = RegSet(rFp).add(rDest) | retvalRegs;
2700 assert((m_state.liveRegs[m_curInst] - expectedLiveRegs).empty());
2701 assert(rFp == rVmFp &&
2702 "free locals helper assumes the frame pointer is rVmFp");
2703 assert(rDest == rVmSp &&
2704 "free locals helper adjusts rVmSp, which must be our dst reg");
2705
2706 if (!numLocals) {
2707 a. lea (rFp[AROFF(m_r)], rDest);
2708 return;
2709 }
2710
2711 // Remove overlap so we don't move registers that are already in the
2712 // saved set.
2713 auto intersectedRegs = spillRegs & retvalRegs;
2714 spillRegs -= intersectedRegs;
2715 retvalRegs -= intersectedRegs;
2716
2717 auto grabPair = [&] (std::pair<PhysReg,PhysReg>& out) {
2718 assert(!retvalRegs.empty() && !spillRegs.empty());
2719 retvalRegs.findFirst(out.first);
2720 spillRegs.findFirst(out.second);
2721 retvalRegs.remove(out.first);
2722 spillRegs.remove(out.second);
2723 };
2724
2725 auto savePairA = std::make_pair(InvalidReg, InvalidReg);
2726 auto savePairB = std::make_pair(InvalidReg, InvalidReg);
2727 if (!retvalRegs.empty()) {
2728 grabPair(savePairA);
2729 if (!retvalRegs.empty()) {
2730 grabPair(savePairB);
2731 }
2732 }
2733
2734 if (savePairA.first != InvalidReg) {
2735 a. movq (savePairA.first, savePairA.second);
2736 if (savePairB.first != InvalidReg) {
2737 a.movq (savePairB.first, savePairB.second);
2738 }
2739 }
2740
2741 auto const target = numLocals > kNumFreeLocalsHelpers
2742 ? m_tx64->m_freeManyLocalsHelper
2743 : m_tx64->m_freeLocalsHelpers[numLocals - 1];
2744
2745 a. subq (0x8, rsp); // For parity; callee does retq $0x8.
2746 a. lea (rFp[-numLocals * sizeof(TypedValue)], rVmSp);
2747 a. call (target);
2748 recordSyncPoint(a);
2749
2750 if (savePairA.first != InvalidReg) {
2751 a. movq (savePairA.second, savePairA.first);
2752 if (savePairB.first != InvalidReg) {
2753 a.movq (savePairB.second, savePairB.first);
2754 }
2755 }
2756 }
2757
2758 static void
2759 tv_release_generic(TypedValue* tv) {
2760 assert(VM::Transl::tx64->stateIsDirty());
2761 assert(tv->m_type >= KindOfString && tv->m_type <= KindOfRef);
2762 g_destructors[typeToDestrIndex(tv->m_type)](tv->m_data.pref);
2763 }
2764
2765 static void
2766 tv_release_typed(RefData* pv, DataType dt) {
2767 assert(VM::Transl::tx64->stateIsDirty());
2768 assert(dt >= KindOfString && dt <= KindOfRef);
2769 g_destructors[typeToDestrIndex(dt)](pv);
2770 }
2771
2772 Address CodeGenerator::getDtorGeneric() {
2773 return (Address)tv_release_generic;
2774 }
2775
2776 Address CodeGenerator::getDtorTyped() {
2777 return (Address)tv_release_typed;
2778 }
2779
2780 //
2781 // This method generates code that checks the static bit and jumps if the bit
2782 // is set. If regIsCount is true, reg contains the _count field. Otherwise,
2783 // it's assumed to contain m_data field.
2784 //
2785 // Return value: the address to be patched with the address to jump to in case
2786 // the static bit is set. If the check is unnecessary, this method retuns NULL.
2787 Address CodeGenerator::cgCheckStaticBit(Type type,
2788 PhysReg reg,
2789 bool regIsCount) {
2790 if (!type.needsStaticBitCheck()) return NULL;
2791
2792 if (regIsCount) {
2793 // reg has the _count value
2794 m_as.cmp_imm32_reg32(RefCountStaticValue, reg);
2795 } else {
2796 // reg has the data pointer
2797 m_as.cmp_imm32_disp_reg32(RefCountStaticValue, FAST_REFCOUNT_OFFSET, reg);
2798 }
2799
2800 Address addrToPatch = m_as.code.frontier;
2801 m_as.jcc8(CC_E, addrToPatch);
2802 return addrToPatch;
2803 }
2804
2805
2806 //
2807 // Using the given dataReg, this method generates code that checks the static
2808 // bit out of dataReg, and emits a DecRef if needed.
2809 // NOTE: the flags are left with the result of the DecRef's subtraction,
2810 // which can then be tested immediately after this.
2811 //
2812 // Return value: the address to be patched if a RefCountedStaticValue check is
2813 // emitted; NULL otherwise.
2814 //
2815 Address CodeGenerator::cgCheckStaticBitAndDecRef(Type type,
2816 PhysReg dataReg,
2817 Block* exit) {
2818 assert(type.maybeCounted());
2819
2820 Address patchStaticCheck = nullptr;
2821 const auto scratchReg = rScratch;
2822
2823 bool canUseScratch = dataReg != scratchReg;
2824
2825 // TODO: run experiments to check whether the 'if' code sequence
2826 // is any better than the 'else' branch below; otherwise, always
2827 // use the 'else' code sequence
2828 if (type.needsStaticBitCheck() && canUseScratch) {
2829 // If we need to check for static value, then load the _count into a
2830 // register to avoid doing two loads. The generated sequence is:
2831 //
2832 // scratchReg = [dataReg + offset(_count)]
2833 // if scratchReg == RefCountStaticValue then skip DecRef
2834 // scratchReg = scratchReg - 1
2835 // ( if exit != NULL, emit:
2836 // jz exit
2837 // )
2838 // [dataReg + offset(_count)] = scratchReg
2839
2840 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2841 emitAssertRefCount(m_as, dataReg);
2842 }
2843 // Load _count in scratchReg
2844 m_as.loadl(dataReg[FAST_REFCOUNT_OFFSET], r32(scratchReg));
2845
2846 // Check for RefCountStaticValue
2847 patchStaticCheck = cgCheckStaticBit(type, scratchReg,
2848 true /* reg has _count */);
2849
2850 // Decrement count and store it back in memory.
2851 // If there's an exit, emit jump to it when _count would get down to 0
2852 m_as.decq(scratchReg);
2853 if (exit) {
2854 emitFwdJcc(CC_E, exit);
2855 }
2856 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2857 // Assert that the ref count is greater than zero
2858 emitAssertFlagsNonNegative(m_as);
2859 }
2860 m_as.store_reg32_disp_reg64(scratchReg, FAST_REFCOUNT_OFFSET, dataReg);
2861
2862 } else {
2863 // Can't use scratch reg, so emit code that operates directly in
2864 // memory. Compared to the sequence above, this will result in one
2865 // extra load, but it has the advantage of producing a instruction
2866 // sequence.
2867 //
2868 // ( if needStaticBitCheck, emit :
2869 // cmp [dataReg + offset(_count)], RefCountStaticValue
2870 // je LabelAfterDecRef
2871 // )
2872 // ( if exit != NULL, emit:
2873 // cmp [dataReg + offset(_count)], 1
2874 // jz exit
2875 // )
2876 // sub [dataReg + offset(_count)], 1
2877
2878 // If necessary, check for RefCountStaticValue
2879 patchStaticCheck = cgCheckStaticBit(type, dataReg,
2880 false /* passing dataReg */);
2881
2882 // If there's an exit, emit jump to it if _count would get down to 0
2883 if (exit) {
2884 m_as.cmp_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET, dataReg);
2885 emitFwdJcc(CC_E, exit);
2886 }
2887 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2888 emitAssertRefCount(m_as, dataReg);
2889 }
2890
2891 // Decrement _count
2892 m_as.decl(dataReg[FAST_REFCOUNT_OFFSET]);
2893
2894 if (RuntimeOption::EvalHHIRGenerateAsserts) {
2895 // Assert that the ref count is not less than zero
2896 emitAssertFlagsNonNegative(m_as);
2897 }
2898 }
2899
2900 return patchStaticCheck;
2901 }
2902
2903
2904 //
2905 // Returns the address to be patched with the address to jump to in case
2906 // the type is not ref-counted.
2907 //
2908 Address CodeGenerator::cgCheckRefCountedType(PhysReg typeReg) {
2909 emitCmpTVType(m_as, KindOfRefCountThreshold, typeReg);
2910 Address addrToPatch = m_as.code.frontier;
2911 m_as.jcc8(CC_LE, addrToPatch);
2912 return addrToPatch;
2913 }
2914
2915 Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset) {
2916 emitCmpTVType(m_as, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]);
2917 Address addrToPatch = m_as.code.frontier;
2918 m_as.jcc8(CC_LE, addrToPatch);
2919 return addrToPatch;
2920 }
2921
2922 //
2923 // Generates dec-ref of a typed value with statically known type.
2924 //
2925 void CodeGenerator::cgDecRefStaticType(Type type,
2926 PhysReg dataReg,
2927 Block* exit,
2928 bool genZeroCheck,
2929 std::function<void(Asm&)>
2930 slowPathWork) {
2931 assert(type != Type::Cell && type != Type::Gen);
2932 assert(type.isKnownDataType());
2933
2934 if (type.notCounted()) return;
2935
2936 // Check for RefCountStaticValue if needed, do the actual DecRef,
2937 // and leave flags set based on the subtract result, which is
2938 // tested below
2939 Address patchStaticCheck;
2940 if (genZeroCheck) {
2941 patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, exit);
2942 } else {
2943 // Set exit as NULL so that the code doesn't jump to error checking.
2944 patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, nullptr);
2945 }
2946
2947 // If not exiting on count down to zero, emit the zero-check and
2948 // release call
2949 if (genZeroCheck && exit == nullptr) {
2950 // Emit jump to m_astubs (to call release) if count got down to zero
2951 unlikelyIfBlock(CC_Z, [&] (Asm& a) {
2952 if (slowPathWork) slowPathWork(a);
2953 // Emit the call to release in m_astubs
2954 cgCallHelper(a, m_tx64->getDtorCall(type.toDataType()),
2955 InvalidReg, InvalidReg, kSyncPoint,
2956 ArgGroup().reg(dataReg));
2957 });
2958 }
2959 if (patchStaticCheck) {
2960 m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
2961 }
2962 }
2963
2964 //
2965 // Generates dec-ref of a typed value with dynamic (statically unknown) type,
2966 // when the type is stored in typeReg.
2967 //
2968 void CodeGenerator::cgDecRefDynamicType(PhysReg typeReg,
2969 PhysReg dataReg,
2970 Block* exit,
2971 bool genZeroCheck) {
2972 // Emit check for ref-counted type
2973 Address patchTypeCheck = cgCheckRefCountedType(typeReg);
2974
2975 // Emit check for RefCountStaticValue and the actual DecRef
2976 Address patchStaticCheck;
2977 if (genZeroCheck) {
2978 patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, exit);
2979 } else {
2980 patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, nullptr);
2981 }
2982
2983 // If not exiting on count down to zero, emit the zero-check and release call
2984 if (genZeroCheck && exit == nullptr) {
2985 // Emit jump to m_astubs (to call release) if count got down to zero
2986 unlikelyIfBlock(CC_Z, [&] (Asm& a) {
2987 // Emit call to release in m_astubs
2988 cgCallHelper(a, getDtorTyped(), InvalidReg, kSyncPoint,
2989 ArgGroup().reg(dataReg).reg(typeReg));
2990 });
2991 }
2992 // Patch checks to jump around the DecRef
2993 if (patchTypeCheck) m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
2994 if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
2995 }
2996
2997 //
2998 // Generates dec-ref of a typed value with dynamic (statically
2999 // unknown) type, when all we have is the baseReg and offset of
3000 // the typed value. This method assumes that baseReg is not the
3001 // scratch register.
3002 //
3003 void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
3004 int64_t offset,
3005 Block* exit) {
3006 auto scratchReg = rScratch;
3007
3008 assert(baseReg != scratchReg);
3009
3010 // Emit check for ref-counted type
3011 Address patchTypeCheck = cgCheckRefCountedType(baseReg, offset);
3012 if (exit == nullptr && RuntimeOption::EvalHHIRGenericDtorHelper) {
3013 {
3014 // This PhysRegSaverParity saves rdi redundantly if
3015 // !liveRegs[m_curInst].contains(rdi), but its
3016 // necessary to maintain stack alignment. We can do better
3017 // by making the helpers adjust the stack for us in the cold
3018 // path, which calls the destructor.
3019 PhysRegSaverParity<0> regSaver(m_as, RegSet(rdi));
3020
3021 /*
3022 * rVmSp is ok here because this is part of the special
3023 * ABI to m_irPopRHelper. We're not using a secret dependency
3024 * on the frame or stack---we're only going to use that ABI if
3025 * we happen to have that register allocated for baseReg.
3026 */
3027 if (offset == 0 && baseReg == rVmSp) {
3028 // Decref'ing top of vm stack, very likely a popR
3029 m_tx64->emitCall(m_as, m_tx64->m_irPopRHelper);
3030 } else {
3031 if (baseReg == rsp) {
3032 // Because we just pushed %rdi, %rsp is 8 bytes below where
3033 // offset is expecting it to be.
3034 offset += sizeof(int64_t);
3035 }
3036 m_as.lea(baseReg[offset], rdi);
3037 m_tx64->emitCall(m_as, m_tx64->m_dtorGenericStub);
3038 }
3039 recordSyncPoint(m_as);
3040 }
3041 if (patchTypeCheck) {
3042 m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
3043 }
3044 return;
3045 }
3046 // Load m_data into the scratch reg
3047 m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
3048
3049 // Emit check for RefCountStaticValue and the actual DecRef
3050 Address patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, scratchReg,
3051 exit);
3052
3053 // If not exiting on count down to zero, emit the zero-check and release call
3054 if (exit == nullptr) {
3055 // Emit jump to m_astubs (to call release) if count got down to zero
3056 unlikelyIfBlock(CC_Z, [&] (Asm& a) {
3057 // Emit call to release in m_astubs
3058 a.lea(baseReg[offset], scratchReg);
3059 cgCallHelper(a, getDtorGeneric(), InvalidReg, kSyncPoint,
3060 ArgGroup().reg(scratchReg));
3061 });
3062 }
3063
3064 // Patch checks to jump around the DecRef
3065 if (patchTypeCheck) m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
3066 if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
3067 }
3068
3069 //
3070 // Generates the dec-ref of a typed value in memory address [baseReg + offset].
3071 // This handles cases where type is either static or dynamic.
3072 //
3073 void CodeGenerator::cgDecRefMem(Type type,
3074 PhysReg baseReg,
3075 int64_t offset,
3076 Block* exit) {
3077 auto scratchReg = rScratch;
3078 assert(baseReg != scratchReg);
3079
3080 if (type.needsReg()) {
3081 // The type is dynamic, but we don't have two registers available
3082 // to load the type and the data.
3083 cgDecRefDynamicTypeMem(baseReg, offset, exit);
3084 } else if (type.maybeCounted()) {
3085 m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
3086 cgDecRefStaticType(type, scratchReg, exit, true);
3087 }
3088 }
3089
3090 void CodeGenerator::cgDecRefMem(IRInstruction* inst) {
3091 assert(inst->getSrc(0)->type().isPtr());
3092 cgDecRefMem(inst->getTypeParam(),
3093 inst->getSrc(0)->getReg(),
3094 inst->getSrc(1)->getValInt(),
3095 inst->getTaken());
3096 }
3097
3098 void CodeGenerator::cgDecRefWork(IRInstruction* inst, bool genZeroCheck) {
3099 SSATmp* src = inst->getSrc(0);
3100 if (!isRefCounted(src)) return;
3101 Block* exit = inst->getTaken();
3102 Type type = src->type();
3103 if (type.isKnownDataType()) {
3104 cgDecRefStaticType(type, src->getReg(), exit, genZeroCheck);
3105 } else {
3106 cgDecRefDynamicType(src->getReg(1),
3107 src->getReg(0),
3108 exit,
3109 genZeroCheck);
3110 }
3111 }
3112
3113 void CodeGenerator::cgDecRef(IRInstruction *inst) {
3114 // DecRef may bring the count to zero, and run the destructor.
3115 // Generate code for this.
3116 assert(!inst->getTaken());
3117 cgDecRefWork(inst, true);
3118 }
3119
3120 void CodeGenerator::cgDecRefNZ(IRInstruction* inst) {
3121 // DecRefNZ cannot bring the count to zero.
3122 // Therefore, we don't generate zero-checking code.
3123 assert(!inst->getTaken());
3124 cgDecRefWork(inst, false);
3125 }
3126
3127 void CodeGenerator::cgDecRefNZOrBranch(IRInstruction* inst) {
3128 assert(inst->getTaken());
3129 cgDecRefWork(inst, true);
3130 }
3131
3132 void CodeGenerator::cgSpillFrame(IRInstruction* inst) {
3133 auto const sp = inst->getSrc(0);
3134 auto const fp = inst->getSrc(1);
3135 auto const func = inst->getSrc(2);
3136 auto const objOrCls = inst->getSrc(3);
3137 auto const magicName = inst->getExtra<SpillFrame>()->invName;
3138 auto const nArgs = inst->getExtra<SpillFrame>()->numArgs;
3139
3140 const int64_t spOffset = -kNumActRecCells * sizeof(Cell);
3141
3142 DEBUG_ONLY bool setThis = true;
3143
3144 auto spReg = sp->getReg();
3145 // actRec->m_this
3146 if (objOrCls->isA(Type::Cls)) {
3147 // store class
3148 if (objOrCls->isConst()) {
3149 m_as.store_imm64_disp_reg64(uintptr_t(objOrCls->getValClass()) | 1,
3150 spOffset + int(AROFF(m_this)),
3151 spReg);
3152 } else {
3153 Reg64 clsPtrReg = objOrCls->getReg();
3154 m_as.movq (clsPtrReg, rScratch);
3155 m_as.orq (1, rScratch);
3156 m_as.storeq(rScratch, spReg[spOffset + int(AROFF(m_this))]);
3157 }
3158 } else if (objOrCls->isA(Type::Obj)) {
3159 // store this pointer
3160 m_as.store_reg64_disp_reg64(objOrCls->getReg(),
3161 spOffset + int(AROFF(m_this)),
3162 spReg);
3163 } else if (objOrCls->isA(Type::Ctx)) {
3164 // Stores either a this pointer or a Cctx -- statically unknown.
3165 Reg64 objOrClsPtrReg = objOrCls->getReg();
3166 m_as.storeq(objOrClsPtrReg, spReg[spOffset + int(AROFF(m_this))]);
3167 } else {
3168 assert(objOrCls->isA(Type::InitNull));
3169 // no obj or class; this happens in FPushFunc
3170 int offset_m_this = spOffset + int(AROFF(m_this));
3171 // When func is either Type::FuncCls or Type::FuncCtx,
3172 // m_this/m_cls will be initialized below
3173 if (!func->isConst() && (func->isA(Type::FuncCtx))) {
3174 // m_this is unioned with m_cls and will be initialized below
3175 setThis = false;
3176 } else {
3177 m_as.store_imm64_disp_reg64(0, offset_m_this, spReg);
3178 }
3179 }
3180 // actRec->m_invName
3181 // ActRec::m_invName is encoded as a pointer with bit kInvNameBit
3182 // set to distinguish it from m_varEnv and m_extrArgs
3183 uintptr_t invName = !magicName
3184 ? 0
3185 : reinterpret_cast<uintptr_t>(magicName) | ActRec::kInvNameBit;
3186 m_as.store_imm64_disp_reg64(invName,
3187 spOffset + int(AROFF(m_invName)),
3188 spReg);
3189 // actRec->m_func and possibly actRec->m_cls
3190 // Note m_cls is unioned with m_this and may overwrite previous value
3191 if (func->type().isNull()) {
3192 assert(func->isConst());
3193 } else if (func->isConst()) {
3194 const Func* f = func->getValFunc();
3195 m_as. mov_imm64_reg((uint64_t)f, rScratch);
3196 m_as.store_reg64_disp_reg64(rScratch,
3197 spOffset + int(AROFF(m_func)),
3198 spReg);
3199 if (func->isA(Type::FuncCtx)) {
3200 // Fill in m_cls if provided with both func* and class*
3201 CG_PUNT(cgAllocActRec);
3202 }
3203 } else {
3204 int offset_m_func = spOffset + int(AROFF(m_func));
3205 m_as.store_reg64_disp_reg64(func->getReg(0),
3206 offset_m_func,
3207 spReg);
3208 if (func->isA(Type::FuncCtx)) {
3209 int offset_m_cls = spOffset + int(AROFF(m_cls));
3210 m_as.store_reg64_disp_reg64(func->getReg(1),
3211 offset_m_cls,
3212 spReg);
3213 setThis = true; /* m_this and m_cls are in a union */
3214 }
3215 }
3216 assert(setThis);
3217 // actRec->m_savedRbp
3218 m_as.store_reg64_disp_reg64(fp->getReg(),
3219 spOffset + int(AROFF(m_savedRbp)),
3220 spReg);
3221
3222 // actRec->m_numArgsAndCtorFlag
3223 m_as.store_imm32_disp_reg(nArgs,
3224 spOffset + int(AROFF(m_numArgsAndCtorFlag)),
3225 spReg);
3226
3227 emitAdjustSp(spReg,
3228 inst->getDst()->getReg(),
3229 spOffset);
3230 }
3231
3232 const Func* loadClassCtor(Class* cls) {
3233 const Func* f = cls->getCtor();
3234 if (UNLIKELY(!(f->attrs() & AttrPublic))) {
3235 VMRegAnchor _;
3236 UNUSED MethodLookup::LookupResult res =
3237 g_vmContext->lookupCtorMethod(f, cls, true /*raise*/);
3238 assert(res == MethodLookup::MethodFoundWithThis);
3239 }
3240 return f;
3241 }
3242
3243 HOT_FUNC_VM ActRec*
3244 irNewInstanceHelper(Class* cls,
3245 int numArgs,
3246 Cell* sp,
3247 ActRec* prevAr) {
3248 Cell* obj = sp - 1; // this is where the newly allocated object will go
3249 ActRec* ar = (ActRec*)(uintptr_t(obj) - sizeof(ActRec));
3250
3251 Instance* newObj = newInstanceHelper(cls, numArgs, ar, prevAr);
3252 // store obj into the stack
3253 obj->m_data.pobj = newObj;
3254 obj->m_type = KindOfObject;
3255 return ar;
3256 }
3257
3258 static inline ALWAYS_INLINE Class* getKnownClass(Class** classCache,
3259 const StringData* clsName) {
3260 Class* cls = *classCache;
3261 if (UNLIKELY(cls == nullptr)) {
3262 // lookupKnownClass does its own VMRegAnchor'ing.
3263 cls = TargetCache::lookupKnownClass<false>(classCache, clsName, true);
3264 assert(*classCache && *classCache == cls);
3265 }
3266 assert(cls);
3267 return cls;
3268 }
3269
3270 Instance* createClHelper(Class* cls, int numArgs, ActRec* ar, TypedValue* sp) {
3271 Instance* newObj = newInstance(cls);
3272 newObj->incRefCount();
3273 return static_cast<c_Closure*>(newObj)->init(numArgs, ar, sp);
3274 }
3275
3276 void CodeGenerator::cgAllocObj(IRInstruction* inst) {
3277 SSATmp* dst = inst->getDst();
3278 SSATmp* cls = inst->getSrc(0);
3279
3280 Instance* (*helper)(Class*) = newInstance;
3281 ArgGroup args;
3282 args.ssa(cls);
3283 cgCallHelper(m_as,
3284 (TCA)helper,
3285 dst,
3286 kSyncPoint,
3287 args);
3288 }
3289
3290 void CodeGenerator::cgInlineCreateCont(IRInstruction* inst) {
3291 auto const& data = *inst->getExtra<InlineCreateCont>();
3292 auto const helper = data.origFunc->isMethod()
3293 ? &VMExecutionContext::createContinuationHelper<true>
3294 : &VMExecutionContext::createContinuationHelper<false>;
3295
3296 cgCallHelper(
3297 m_as,
3298 reinterpret_cast<TCA>(helper),
3299 inst->getDst(),
3300 kSyncPoint,
3301 ArgGroup()
3302 .immPtr(data.origFunc)
3303 .immPtr(data.genFunc)
3304 .ssa(inst->getSrc(0))
3305 .immPtr(nullptr) // getArgs array
3306 // Deliberately ignoring frameStaticClass parameter, because
3307 // it's unused if we have a $this pointer, and we don't inline
3308 // functions with a null $this.
3309 );
3310 if (data.origFunc->isMethod()) {
3311 // We can't support a null $this.
3312 assert(inst->getSrc(0)->isA(Type::Obj));
3313 }
3314 }
3315
3316 void CodeGenerator::cgCallArray(IRInstruction* inst) {
3317 Offset pc = inst->getExtra<CallArray>()->pc;
3318 Offset after = inst->getExtra<CallArray>()->after;
3319
3320 ArgGroup args;
3321 args.imm(pc).imm(after);
3322
3323 // fCallArrayHelper makes the actual call by smashing its return address.
3324 cgCallHelper(m_as, (TCA)TranslatorX64::fCallArrayHelper,
3325 nullptr, kSyncPoint, args);
3326 }
3327
3328 void CodeGenerator::cgCall(IRInstruction* inst) {
3329 SSATmp* actRec = inst->getSrc(0);
3330 SSATmp* returnBcOffset = inst->getSrc(1);
3331 SSATmp* func = inst->getSrc(2);
3332 SrcRange args = inst->getSrcs().subpiece(3);
3333 int32_t numArgs = args.size();
3334
3335 auto spReg = actRec->getReg();
3336 // put all outgoing arguments onto the VM stack
3337 int64_t adjustment = (-(int64_t)numArgs) * sizeof(Cell);
3338 for (int32_t i = 0; i < numArgs; i++) {
3339 // Type::None here means that the simplifier proved that the value
3340 // matches the value already in memory, thus the store is redundant.
3341 if (args[i]->type() != Type::None) {
3342 cgStore(spReg, -(i + 1) * sizeof(Cell), args[i]);
3343 }
3344 }
3345 // store the return bytecode offset into the outgoing actrec
3346 uint64_t returnBc = returnBcOffset->getValInt();
3347 m_as.store_imm32_disp_reg(returnBc, AROFF(m_soff), spReg);
3348 if (adjustment != 0) {
3349 m_as.add_imm32_reg64(adjustment, spReg);
3350 }
3351
3352 assert(m_state.lastMarker);
3353 SrcKey srcKey = SrcKey(m_state.lastMarker->func, m_state.lastMarker->bcOff);
3354 bool isImmutable = (func->isConst() && !func->type().isNull());
3355 const Func* funcd = isImmutable ? func->getValFunc() : nullptr;
3356 assert(&m_as == &m_tx64->getAsm());
3357 int32_t adjust = m_tx64->emitBindCall(srcKey, funcd, numArgs);
3358 if (adjust) {
3359 m_as.addq (adjust, rVmSp);
3360 }
3361 }
3362
3363 void CodeGenerator::cgCastStk(IRInstruction *inst) {
3364 Type type = inst->getTypeParam();
3365 SSATmp* sp = inst->getSrc(0);
3366 uint32_t offset = inst->getExtra<CastStk>()->offset;
3367 PhysReg spReg = sp->getReg();
3368
3369 ArgGroup args;
3370 args.addr(spReg, cellsToBytes(offset));
3371
3372 TCA tvCastHelper;
3373 if (type.subtypeOf(Type::Bool)) {
3374 tvCastHelper = (TCA)tvCastToBooleanInPlace;
3375 } else if (type.subtypeOf(Type::Int)) {
3376 // if casting to integer, pass 10 as the base for the conversion
3377 args.imm(10);
3378 tvCastHelper = (TCA)tvCastToInt64InPlace;
3379 } else if (type.subtypeOf(Type::Dbl)) {
3380 tvCastHelper = (TCA)tvCastToDoubleInPlace;
3381 } else if (type.subtypeOf(Type::Arr)) {
3382 tvCastHelper = (TCA)tvCastToArrayInPlace;
3383 } else if (type.subtypeOf(Type::Str)) {
3384 tvCastHelper = (TCA)tvCastToStringInPlace;
3385 } else if (type.subtypeOf(Type::Obj)) {
3386 tvCastHelper = (TCA)tvCastToObjectInPlace;
3387 } else {
3388 not_reached();
3389 }
3390 cgCallHelper(m_as, tvCastHelper, nullptr,
3391 kSyncPoint, args, DestType::None);
3392 }
3393
3394 void CodeGenerator::cgCallBuiltin(IRInstruction* inst) {
3395 SSATmp* f = inst->getSrc(0);
3396 auto args = inst->getSrcs().subpiece(1);
3397 int32_t numArgs = args.size();
3398 SSATmp* dst = inst->getDst();
3399 auto dstReg = dst->getReg(0);
3400 auto dstType = dst->getReg(1);
3401 Type returnType = inst->getTypeParam();
3402
3403 const Func* func = f->getValFunc();
3404 DataType funcReturnType = func->returnType();
3405 int returnOffset = HHIR_MISOFF(tvBuiltinReturn);
3406
3407 // RSP points to the MInstrState we need to use.
3408 // workaround the fact that rsp moves when we spill registers around call
3409 PhysReg misReg = rScratch;
3410 emitMovRegReg(m_as, reg::rsp, misReg);
3411
3412 ArgGroup callArgs;
3413 if (isCppByRef(funcReturnType)) {
3414 // first arg is pointer to storage for that return value
3415 if (isSmartPtrRef(funcReturnType)) {
3416 returnOffset += TVOFF(m_data);
3417 }
3418 // misReg is pointing to an MInstrState struct on the C stack. Pass
3419 // the address of tvBuiltinReturn to the native function as the location
3420 // it can construct the return Array, String, Object, or Variant.
3421 callArgs.addr(misReg, returnOffset); // &misReg[returnOffset]
3422 }
3423
3424 // non-pointer args are plain values passed by value. String, Array,
3425 // Object, and Variant are passed by const&, ie a pointer to stack memory
3426 // holding the value, so expect PtrToT types for these.
3427 // Pointers to smartptr types (String, Array, Object) need adjusting to
3428 // point to &ptr->m_data.
3429 for (int i = 0; i < numArgs; i++) {
3430 const Func::ParamInfo& pi = func->params()[i];
3431 if (TVOFF(m_data) && isSmartPtrRef(pi.builtinType())) {
3432 assert(args[i]->type().isPtr() && args[i]->getReg() != InvalidReg);
3433 callArgs.addr(args[i]->getReg(), TVOFF(m_data));
3434 } else {
3435 callArgs.ssa(args[i]);
3436 }
3437 }
3438
3439 // if the return value is returned by reference, we don't need the
3440 // return value from this call since we know where the value is.
3441 cgCallHelper(m_as, Transl::Call((TCA)func->nativeFuncPtr()),
3442 isCppByRef(funcReturnType) ? InvalidReg : dstReg,
3443 kSyncPoint, callArgs);
3444
3445 // load return value from builtin
3446 // for primitive return types (int, bool), the return value
3447 // is already in dstReg (the builtin call returns in rax). For return
3448 // by reference (String, Object, Array, Variant), the builtin writes the
3449 // return value into MInstrState::tvBuiltinReturn TV, from where it
3450 // has to be tested and copied.
3451 if (dstReg == InvalidReg || returnType.isSimpleType()) {
3452 return;
3453 }
3454 // after the call, RSP is back pointing to MInstrState and rSratch
3455 // has been clobberred.
3456 misReg = rsp;
3457
3458 if (returnType.isReferenceType()) {
3459 assert(isCppByRef(funcReturnType) && isSmartPtrRef(funcReturnType));
3460 // return type is String, Array, or Object; fold nullptr to KindOfNull
3461 m_as. loadq (misReg[returnOffset], dstReg);
3462 emitLoadImm(m_as, returnType.toDataType(), dstType);
3463 emitLoadImm(m_as, KindOfNull, rScratch);
3464 m_as. testq (dstReg, dstReg);
3465 m_as. cmov_reg64_reg64 (CC_Z, rScratch, dstType);
3466 return;
3467 }
3468 if (returnType.subtypeOf(Type::Cell)
3469 || returnType.subtypeOf(Type::BoxedCell)) {
3470 // return type is Variant; fold KindOfUninit to KindOfNull
3471 assert(isCppByRef(funcReturnType) && !isSmartPtrRef(funcReturnType));
3472 assert(misReg != dstType);
3473 emitLoadTVType(m_as, misReg[returnOffset + TVOFF(m_type)], dstType);
3474 m_as. loadq (misReg[returnOffset + TVOFF(m_data)], dstReg);
3475 emitLoadImm(m_as, KindOfNull, rScratch);
3476 static_assert(KindOfUninit == 0, "KindOfUninit must be 0 for test");
3477 m_as. testb (rbyte(dstType), rbyte(dstType));
3478 m_as. cmov_reg64_reg64 (CC_Z, rScratch, dstType);
3479 return;
3480 }
3481 not_reached();
3482 }
3483
3484 void CodeGenerator::cgSpillStack(IRInstruction* inst) {
3485 SSATmp* dst = inst->getDst();
3486 SSATmp* sp = inst->getSrc(0);
3487 auto const spDeficit = inst->getSrc(1)->getValInt();
3488 auto const spillVals = inst->getSrcs().subpiece(2);
3489 auto const numSpillSrcs = spillVals.size();
3490 auto const dstReg = dst->getReg();
3491 auto const spReg = sp->getReg();
3492 auto const spillCells = spillValueCells(inst);
3493
3494 int64_t adjustment = (spDeficit - spillCells) * sizeof(Cell);
3495 for (uint32_t i = 0; i < numSpillSrcs; ++i) {
3496 const int64_t offset = i * sizeof(Cell) + adjustment;
3497 if (spillVals[i]->type() == Type::None) {
3498 // The simplifier detected that we're storing the same value
3499 // already in there.
3500 continue;
3501 }
3502
3503 auto* val = spillVals[i];
3504 auto* inst = val->inst();
3505 while (inst->isPassthrough()) {
3506 inst = inst->getPassthroughValue()->inst();
3507 }
3508 // If our value came from a LdStack on the same sp and offset,
3509 // we don't need to spill it.
3510 if (inst->op() == LdStack && inst->getSrc(0) == sp &&
3511 inst->getExtra<LdStack>()->offset * sizeof(Cell) == offset) {
3512 FTRACE(1, "{}: Not spilling spill value {} from {}\n",
3513 __func__, i, inst->toString());
3514 } else {
3515 cgStore(spReg, offset, val);
3516 }
3517 }
3518
3519 emitAdjustSp(spReg, dstReg, adjustment);
3520 }
3521
3522 void CodeGenerator::emitAdjustSp(PhysReg spReg,
3523 PhysReg dstReg,
3524 int64_t adjustment /* bytes */) {
3525 if (adjustment != 0) {
3526 if (dstReg != spReg) {
3527 m_as. lea (spReg[adjustment], dstReg);
3528 } else {
3529 m_as. addq (adjustment, dstReg);
3530 }
3531 } else {
3532 emitMovRegReg(m_as, spReg, dstReg);
3533 }
3534 }
3535
3536 void CodeGenerator::cgNativeImpl(IRInstruction* inst) {
3537 SSATmp* func = inst->getSrc(0);
3538 SSATmp* fp = inst->getSrc(1);
3539
3540 assert(func->isConst());
3541 assert(func->type() == Type::Func);
3542
3543 BuiltinFunction builtinFuncPtr = func->getValFunc()->builtinFuncPtr();
3544 emitMovRegReg(m_as, fp->getReg(), argNumToRegName[0]);
3545 m_as.call((TCA)builtinFuncPtr);
3546 recordSyncPoint(m_as);
3547 }
3548
3549 void CodeGenerator::cgLdThis(IRInstruction* inst) {
3550 SSATmp* dst = inst->getDst();
3551 SSATmp* src = inst->getSrc(0);
3552 Block* label = inst->getTaken();
3553 // mov dst, [fp + 0x20]
3554 auto dstReg = dst->getReg();
3555
3556 // the destination of LdThis could be dead but the instruction
3557 // itself still useful because of the checks that it does (if it has
3558 // a label). So we need to make sure there is a dstReg for this
3559 // instruction.
3560 if (dstReg != InvalidReg) {
3561 // instruction's result is not dead
3562 m_as.loadq(src->getReg()[AROFF(m_this)], dstReg);
3563 }
3564 if (label == NULL) return; // no need to perform its checks
3565 if (dstReg != InvalidReg) {
3566 // test 0x01, dst
3567 m_as.testb(1, rbyte(dstReg));
3568 } else {
3569 m_as.testb(1, src->getReg()[AROFF(m_this)]);
3570 }
3571 // jnz label
3572 emitFwdJcc(CC_NZ, label);
3573 }
3574
3575 static void emitLdClsCctx(CodeGenerator::Asm& a,
3576 PhysReg srcReg,
3577 PhysReg dstReg) {
3578 emitMovRegReg(a, srcReg, dstReg);
3579 a. decq(dstReg);
3580 }
3581
3582 void CodeGenerator::cgLdClsCtx(IRInstruction* inst) {
3583 PhysReg srcReg = inst->getSrc(0)->getReg();
3584 PhysReg dstReg = inst->getDst()->getReg();
3585 // Context could be either a this object or a class ptr
3586 m_as. testb(1, rbyte(srcReg));
3587 ifThenElse(CC_NZ,
3588 [&] { emitLdClsCctx(m_as, srcReg, dstReg); }, // ctx is a class
3589 [&] { emitLdObjClass(m_as, srcReg, dstReg); } // ctx is this ptr
3590 );
3591 }
3592
3593 void CodeGenerator::cgLdClsCctx(IRInstruction* inst) {
3594 PhysReg srcReg = inst->getSrc(0)->getReg();
3595 PhysReg dstReg = inst->getDst()->getReg();
3596 emitLdClsCctx(m_as, srcReg, dstReg);
3597 }
3598
3599 void CodeGenerator::cgLdCtx(IRInstruction* inst) {
3600 PhysReg dstReg = inst->getDst()->getReg();
3601 PhysReg srcReg = inst->getSrc(0)->getReg();
3602 if (dstReg != InvalidReg) {
3603 m_as.loadq(srcReg[AROFF(m_this)], dstReg);
3604 }
3605 }
3606
3607 void CodeGenerator::cgLdCctx(IRInstruction* inst) {
3608 return cgLdCtx(inst);
3609 }
3610
3611 void CodeGenerator::cgLdConst(IRInstruction* inst) {
3612 auto const dstReg = inst->getDst()->getReg();
3613 auto const val = inst->getExtra<LdConst>()->as<uintptr_t>();
3614 if (dstReg == InvalidReg) return;
3615 emitLoadImm(m_as, val, dstReg);
3616 }
3617
3618 void CodeGenerator::cgLdARFuncPtr(IRInstruction* inst) {
3619 SSATmp* dst = inst->getDst();
3620 SSATmp* baseAddr = inst->getSrc(0);
3621 SSATmp* offset = inst->getSrc(1);
3622
3623 auto dstReg = dst->getReg();
3624 auto baseReg = baseAddr->getReg();
3625
3626 assert(offset->isConst());
3627
3628 m_as.load_reg64_disp_reg64(baseReg,
3629 offset->getValInt() + AROFF(m_func),
3630 dstReg);
3631 }
3632
3633 void CodeGenerator::cgLdContLocalsPtr(IRInstruction* inst) {
3634 auto rCont = inst->getSrc(0)->getReg();
3635 auto rLocals = inst->getDst()->getReg();
3636 m_as. loadl (rCont[CONTOFF(m_localsOffset)], r32(rLocals));
3637 m_as. addq (rCont, rLocals);
3638 }
3639
3640 static int getNativeTypeSize(Type type) {
3641 if (type.subtypeOf(Type::Int | Type::Func)) return sz::qword;
3642 if (type.subtypeOf(Type::Bool)) return sz::byte;
3643 not_implemented();
3644 }
3645
3646 void CodeGenerator::cgLdRaw(IRInstruction* inst) {
3647 SSATmp* dest = inst->getDst();
3648 SSATmp* addr = inst->getSrc(0);
3649 SSATmp* offset = inst->getSrc(1);
3650
3651 assert(!(dest->isConst()));
3652
3653 Reg64 addrReg = addr->getReg();
3654 PhysReg destReg = dest->getReg();
3655
3656 if (addr->isConst()) {
3657 addrReg = rScratch;
3658 emitLoadImm(m_as, addr->getValRawInt(), addrReg);
3659 }
3660
3661 if (offset->isConst()) {
3662 assert(offset->type() == Type::Int);
3663 int64_t kind = offset->getValInt();
3664 RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
3665 int ldSize = slot.getSize();
3666 int64_t off = slot.getOffset();
3667 if (ldSize == sz::qword) {
3668 m_as.loadq (addrReg[off], destReg);
3669 } else if (ldSize == sz::dword) {
3670 m_as.loadl (addrReg[off], r32(destReg));
3671 } else {
3672 assert(ldSize == sz::byte);
3673 m_as.loadzbl (addrReg[off], r32(destReg));
3674 }
3675 } else {
3676 int ldSize = getNativeTypeSize(dest->type());
3677 Reg64 offsetReg = r64(offset->getReg());
3678 if (ldSize == sz::qword) {
3679 m_as.loadq (addrReg[offsetReg], destReg);
3680 } else {
3681 // Not yet supported by our assembler
3682 assert(ldSize == sz::byte);
3683 not_implemented();
3684 }
3685 }
3686 }
3687
3688 void CodeGenerator::cgStRaw(IRInstruction* inst) {
3689 auto baseReg = inst->getSrc(0)->getReg();
3690 int64_t kind = inst->getSrc(1)->getValInt();
3691 SSATmp* value = inst->getSrc(2);
3692
3693 RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
3694 assert(value->type().equals(slot.type()));
3695 int stSize = slot.getSize();
3696 int64_t off = slot.getOffset();
3697 auto dest = baseReg[off];
3698
3699 if (value->isConst()) {
3700 if (stSize == sz::qword) {
3701 m_as.storeq(value->getValRawInt(), dest);
3702 } else if (stSize == sz::dword) {
3703 m_as.storel(value->getValRawInt(), dest);
3704 } else {
3705 assert(stSize == sz::byte);
3706 m_as.storeb(value->getValBool(), dest);
3707 }
3708 } else {
3709 if (stSize == sz::qword) {
3710 m_as.storeq(r64(value->getReg()), dest);
3711 } else if (stSize == sz::dword) {
3712 m_as.storel(r32(value->getReg()), dest);
3713 } else {
3714 assert(stSize == sz::byte);
3715 m_as.storeb(rbyte(value->getReg()), dest);
3716 }
3717 }
3718 }
3719
3720 void CodeGenerator::cgLdStaticLocCached(IRInstruction* inst) {
3721 auto ch = inst->getSrc(0)->getValRawInt();
3722 auto outReg = inst->getDst()->getReg();
3723
3724 m_as.loadq (rVmTl[ch], outReg);
3725 m_as.testq (outReg, outReg);
3726 emitFwdJcc(m_as, CC_Z, inst->getTaken());
3727 }
3728
3729 // If label is set and type is not Gen, this method generates a check
3730 // that bails to the label if the loaded typed value doesn't match type.
3731 void CodeGenerator::cgLoadTypedValue(PhysReg base,
3732 int64_t off,
3733 IRInstruction* inst) {
3734 Block* label = inst->getTaken();
3735 Type type = inst->getTypeParam();
3736 SSATmp* dst = inst->getDst();
3737
3738 assert(type == dst->type());
3739 assert(type.needsReg());
3740 auto valueDstReg = dst->getReg(0);
3741 auto typeDstReg = dst->getReg(1);
3742 if (valueDstReg == InvalidReg && typeDstReg == InvalidReg &&
3743 (label == nullptr || type == Type::Gen)) {
3744 // a dead load
3745 return;
3746 }
3747 bool useScratchReg = (base == typeDstReg && valueDstReg != InvalidReg);
3748 if (useScratchReg) {
3749 // Save base to rScratch, because base will be overwritten.
3750 m_as.mov_reg64_reg64(base, reg::rScratch);
3751 }
3752
3753 // Load type if it's not dead
3754 if (typeDstReg != InvalidReg) {
3755 emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg);
3756 if (label) {
3757 // Check type needed
3758 emitGuardType(typeDstReg, inst);
3759 }
3760 } else if (label) {
3761 // Check type needed
3762 cgGuardTypeCell(base, off, inst);
3763 }
3764
3765 // Load value if it's not dead
3766 if (valueDstReg == InvalidReg) return;
3767
3768 if (useScratchReg) {
3769 m_as.loadq(reg::rScratch[off + TVOFF(m_data)], valueDstReg);
3770 } else {
3771 m_as.loadq(base[off + TVOFF(m_data)], valueDstReg);
3772 }
3773 }
3774
3775 void CodeGenerator::cgStoreTypedValue(PhysReg base,
3776 int64_t off,
3777 SSATmp* src) {
3778 assert(src->type().needsReg());
3779 m_as.storeq(src->getReg(0), base[off + TVOFF(m_data)]);
3780 emitStoreTVType(m_as, src->getReg(1), base[off + TVOFF(m_type)]);
3781 }
3782
3783 void CodeGenerator::cgStore(PhysReg base,
3784 int64_t off,
3785 SSATmp* src,
3786 bool genStoreType) {
3787 Type type = src->type();
3788 if (type.needsReg()) {
3789 cgStoreTypedValue(base, off, src);
3790 return;
3791 }
3792 // store the type
3793 if (genStoreType) {
3794 emitStoreTVType(m_as, type.toDataType(), base[off + TVOFF(m_type)]);
3795 }
3796 if (type.isNull()) {
3797 // no need to store a value for null or uninit
3798 return;
3799 }
3800 if (src->isConst()) {
3801 int64_t val = 0;
3802 if (type.subtypeOf(Type::Bool | Type::Int | Type::Dbl |
3803 Type::Arr | Type::StaticStr | Type::Cls)) {
3804 val = src->getValBits();
3805 } else {
3806 not_reached();
3807 }
3808 m_as.storeq(val, base[off + TVOFF(m_data)]);
3809 } else {
3810 zeroExtendIfBool(m_as, src);
3811 m_as.storeq(src->getReg(), base[off + TVOFF(m_data)]);
3812 }
3813 }
3814
3815 void CodeGenerator::emitGuardOrFwdJcc(IRInstruction* inst, ConditionCode cc) {
3816 if (cc == CC_None) return;
3817 Block* label = inst->getTaken();
3818 if (inst && inst->getTCA() == kIRDirectGuardActive) {
3819 if (dumpIREnabled()) {
3820 m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen);
3821 inst->setTCA(m_as.code.frontier);
3822 }
3823 // Get the SrcKey for the dest
3824 SrcKey destSK(getCurFunc(), m_curTrace->getBcOff());
3825 SrcRec* destSR = m_tx64->getSrcRec(destSK);
3826 m_tx64->emitFallbackCondJmp(m_as, *destSR, cc);
3827 } else {
3828 emitFwdJcc(cc, label);
3829 }
3830 }
3831
3832 void CodeGenerator::cgLoad(PhysReg base,
3833 int64_t off,
3834 IRInstruction* inst) {
3835 Type type = inst->getTypeParam();
3836 if (type.needsReg()) {
3837 return cgLoadTypedValue(base, off, inst);
3838 }
3839 Block* label = inst->getTaken();
3840 if (label != NULL) {
3841 cgGuardTypeCell(base, off, inst);
3842 }
3843 if (type.isNull()) return; // these are constants
3844 auto dstReg = inst->getDst()->getReg();
3845 // if dstReg == InvalidReg then the value of this load is dead
3846 if (dstReg == InvalidReg) return;
3847
3848 if (type == Type::Bool) {
3849 m_as.load_reg64_disp_reg32(base, off + TVOFF(m_data), dstReg);
3850 } else {
3851 m_as.load_reg64_disp_reg64(base, off + TVOFF(m_data), dstReg);
3852 }
3853 }
3854
3855 void CodeGenerator::cgLdProp(IRInstruction* inst) {
3856 cgLoad(inst->getSrc(0)->getReg(), inst->getSrc(1)->getValInt(), inst);
3857 }
3858
3859 void CodeGenerator::cgLdMem(IRInstruction * inst) {
3860 cgLoad(inst->getSrc(0)->getReg(), inst->getSrc(1)->getValInt(), inst);
3861 }
3862
3863 void CodeGenerator::cgLdRef(IRInstruction* inst) {
3864 cgLoad(inst->getSrc(0)->getReg(), RefData::tvOffset(), inst);
3865 }
3866
3867 void CodeGenerator::recordSyncPoint(Asm& as,
3868 SyncOptions sync /* = kSyncPoint */) {
3869 assert(m_state.lastMarker);
3870
3871 Offset stackOff = m_state.lastMarker->stackOff;
3872 switch (sync) {
3873 case kSyncPointAdjustOne:
3874 stackOff -= 1;
3875 break;
3876 case kSyncPoint:
3877 break;
3878 case kNoSyncPoint:
3879 assert(0);
3880 }
3881
3882 Offset pcOff = m_state.lastMarker->bcOff - m_state.lastMarker->func->base();
3883
3884 FTRACE(3, "IR recordSyncPoint: {} {} {}\n", as.code.frontier, pcOff,
3885 stackOff);
3886 m_tx64->recordSyncPoint(as, pcOff, stackOff);
3887 }
3888
3889 void CodeGenerator::cgLdAddr(IRInstruction* inst) {
3890 auto base = inst->getSrc(0)->getReg();
3891 int64_t offset = inst->getSrc(1)->getValInt();
3892 m_as.lea (base[offset], inst->getDst()->getReg());
3893 }
3894
3895 void CodeGenerator::cgLdLoc(IRInstruction* inst) {
3896 cgLoad(inst->getSrc(0)->getReg(),
3897 getLocalOffset(inst->getExtra<LdLoc>()->locId),
3898 inst);
3899 }
3900
3901 void CodeGenerator::cgLdLocAddr(IRInstruction* inst) {
3902 auto const fpReg = inst->getSrc(0)->getReg();
3903 auto const offset = getLocalOffset(inst->getExtra<LdLocAddr>()->locId);
3904 if (inst->getDst()->hasReg()) {
3905 m_as.lea(fpReg[offset], inst->getDst()->getReg());
3906 }
3907 }
3908
3909 void CodeGenerator::cgLdStackAddr(IRInstruction* inst) {
3910 auto const base = inst->getSrc(0)->getReg();
3911 auto const offset = cellsToBytes(inst->getExtra<LdStackAddr>()->offset);
3912 m_as.lea (base[offset], inst->getDst()->getReg());
3913 }
3914
3915 void CodeGenerator::cgLdStack(IRInstruction* inst) {
3916 assert(inst->getTaken() == nullptr);
3917 cgLoad(inst->getSrc(0)->getReg(),
3918 cellsToBytes(inst->getExtra<LdStack>()->offset),
3919 inst);
3920 }
3921
3922 void CodeGenerator::cgGuardStk(IRInstruction* inst) {
3923 cgGuardTypeCell(inst->getSrc(0)->getReg(),
3924 cellsToBytes(inst->getExtra<GuardStk>()->offset),
3925 inst);
3926 }
3927
3928 void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
3929 cgGuardTypeCell(inst->getSrc(0)->getReg(),
3930 getLocalOffset(inst->getExtra<GuardLoc>()->locId),
3931 inst);
3932 }
3933
3934 void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) {
3935 assert(inst->getDst()->type() == Type::PtrToCell);
3936 assert(inst->getDst()->getReg() == rsp);
3937 }
3938
3939 void CodeGenerator::cgGuardType(IRInstruction* inst) {
3940 Type type = inst->getTypeParam();
3941 SSATmp* src = inst->getSrc(0);
3942 auto srcTypeReg = src->getReg(1);
3943 assert(srcTypeReg != InvalidReg);
3944
3945 ConditionCode cc;
3946 cc = emitTypeTest(type, srcTypeReg, true);
3947 emitFwdJcc(cc, inst->getTaken());
3948
3949 auto dstReg = inst->getDst()->getReg();
3950 if (dstReg != InvalidReg) {
3951 emitMovRegReg(m_as, src->getReg(0), dstReg);
3952 }
3953 }
3954
3955 void CodeGenerator::cgGuardRefs(IRInstruction* inst) {
3956 assert(inst->getNumSrcs() == 6);
3957
3958 SSATmp* funcPtrTmp = inst->getSrc(0);
3959 SSATmp* nParamsTmp = inst->getSrc(1);
3960 SSATmp* bitsPtrTmp = inst->getSrc(2);
3961 SSATmp* firstBitNumTmp = inst->getSrc(3);
3962 SSATmp* mask64Tmp = inst->getSrc(4);
3963 SSATmp* vals64Tmp = inst->getSrc(5);
3964 Block* exitLabel = inst->getTaken();
3965
3966 // Get values in place
3967 assert(funcPtrTmp->type() == Type::Func);
3968 auto funcPtrReg = funcPtrTmp->getReg();
3969 assert(funcPtrReg != InvalidReg);
3970
3971 assert(nParamsTmp->type() == Type::Int);
3972 auto nParamsReg = nParamsTmp->getReg();
3973 assert(nParamsReg != InvalidReg);
3974
3975 assert(bitsPtrTmp->type() == Type::Int);
3976 auto bitsPtrReg = bitsPtrTmp->getReg();
3977 assert(bitsPtrReg != InvalidReg);
3978
3979 assert(firstBitNumTmp->isConst() && firstBitNumTmp->type() == Type::Int);
3980 uint32_t firstBitNum = (uint32_t)(firstBitNumTmp->getValInt());
3981
3982 assert(mask64Tmp->type() == Type::Int);
3983 assert(mask64Tmp->inst()->op() == LdConst);
3984 auto mask64Reg = mask64Tmp->getReg();
3985 assert(mask64Reg != InvalidReg);
3986 int64_t mask64 = mask64Tmp->getValInt();
3987
3988 assert(vals64Tmp->type() == Type::Int);
3989 assert(vals64Tmp->inst()->op() == LdConst);
3990 auto vals64Reg = vals64Tmp->getReg();
3991 assert(vals64Reg != InvalidReg);
3992 int64_t vals64 = vals64Tmp->getValInt();
3993
3994 auto thenBody = [&] {
3995 auto bitsValReg = rScratch;
3996 // Load the bit values in bitValReg:
3997 // bitsValReg <- [bitsValPtr + (firstBitNum / 64)]
3998 m_as.load_reg64_disp_reg64(bitsPtrReg, sizeof(uint64_t) * (firstBitNum / 64),
3999 bitsValReg);
4000 // bitsValReg <- bitsValReg & mask64
4001 m_as.and_reg64_reg64(mask64Reg, bitsValReg);
4002
4003 // If bitsValReg != vals64Reg, then goto Exit
4004 m_as.cmp_reg64_reg64(bitsValReg, vals64Reg);
4005 emitFwdJcc(CC_NE, exitLabel);
4006 };
4007
4008 // If few enough args...
4009 m_as.cmp_imm32_reg32(firstBitNum + 1, nParamsReg);
4010 if (vals64 == 0 && mask64 == 0) {
4011 ifThen(m_as, CC_NL, thenBody);
4012 } else if (vals64 != 0 && vals64 != mask64) {
4013 emitFwdJcc(CC_L, exitLabel);
4014 thenBody();
4015 } else if (vals64 != 0) {
4016 ifThenElse(CC_NL, thenBody, /* else */ [&] {
4017 // If not special builtin...
4018 m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
4019 emitFwdJcc(CC_Z, exitLabel);
4020 });
4021 } else {
4022 ifThenElse(CC_NL, thenBody, /* else */ [&] {
4023 m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
4024 emitFwdJcc(CC_NZ, exitLabel);
4025 });
4026 }
4027 }
4028
4029 void CodeGenerator::cgLdPropAddr(IRInstruction* inst) {
4030 SSATmp* dst = inst->getDst();
4031 SSATmp* obj = inst->getSrc(0);
4032 SSATmp* prop = inst->getSrc(1);
4033
4034 assert(prop->isConst() && prop->type() == Type::Int);
4035
4036 auto dstReg = dst->getReg();
4037 auto objReg = obj->getReg();
4038
4039 assert(objReg != InvalidReg);
4040 assert(dstReg != InvalidReg);
4041
4042 int64_t offset = prop->getValInt();
4043 m_as.lea_reg64_disp_reg64(objReg, offset, dstReg);
4044 }
4045
4046 void CodeGenerator::cgLdClsMethod(IRInstruction* inst) {
4047 SSATmp* dst = inst->getDst();
4048 SSATmp* cls = inst->getSrc(0);
4049 SSATmp* mSlot = inst->getSrc(1);
4050
4051 assert(cls->type() == Type::Cls);
4052 assert(mSlot->isConst() && mSlot->type() == Type::Int);
4053 uint64_t mSlotInt64 = mSlot->getValRawInt();
4054 // We're going to multiply mSlotVal by sizeof(Func*) and use
4055 // it as a 32-bit offset (methOff) below.
4056 if (mSlotInt64 > (std::numeric_limits<uint32_t>::max() / sizeof(Func*))) {
4057 CG_PUNT(cgLdClsMethod_large_offset);
4058 }
4059 int32_t mSlotVal = (uint32_t) mSlotInt64;
4060
4061 Reg64 dstReg = dst->getReg();
4062 assert(dstReg != InvalidReg);
4063
4064 Reg64 clsReg = cls->getReg();
4065 if (clsReg == InvalidReg) {
4066 CG_PUNT(LdClsMethod);
4067 }
4068
4069 Offset vecOff = Class::getMethodsOffset() + Class::MethodMap::vecOff();
4070 int32_t methOff = mSlotVal * sizeof(Func*);
4071 m_as.loadq(clsReg[vecOff], dstReg);
4072 m_as.loadq(dstReg[methOff], dstReg);
4073 }
4074
4075 void CodeGenerator::cgLdClsMethodCache(IRInstruction* inst) {
4076 SSATmp* dst = inst->getDst();
4077 SSATmp* className = inst->getSrc(0);
4078 SSATmp* methodName = inst->getSrc(1);
4079 SSATmp* baseClass = inst->getSrc(2);
4080 Block* label = inst->getTaken();
4081
4082 // Stats::emitInc(a, Stats::TgtCache_StaticMethodHit);
4083 const StringData* cls = className->getValStr();
4084 const StringData* method = methodName->getValStr();
4085 auto const ne = baseClass->getValNamedEntity();
4086 TargetCache::CacheHandle ch =
4087 TargetCache::StaticMethodCache::alloc(cls,
4088 method,
4089 getContextName(getCurClass()));
4090 auto funcDestReg = dst->getReg(0);
4091 auto classDestReg = dst->getReg(1);
4092 auto offsetof_func = offsetof(TargetCache::StaticMethodCache, m_func);
4093 auto offsetof_cls = offsetof(TargetCache::StaticMethodCache, m_cls);
4094
4095 assert(funcDestReg != InvalidReg && classDestReg != InvalidReg);
4096 // Attempt to retrieve the func* and class* from cache
4097 m_as.loadq(rVmTl[ch + offsetof_func], funcDestReg);
4098 m_as.loadq(rVmTl[ch + offsetof_cls], classDestReg);
4099 m_as.testq(funcDestReg, funcDestReg);
4100 // May have retrieved a NULL from the cache
4101 // handle case where method is not entered in the cache
4102 unlikelyIfBlock(CC_E, [&] (Asm& a) {
4103 if (false) { // typecheck
4104 const UNUSED Func* f = StaticMethodCache::lookupIR(ch, ne, cls, method);
4105 }
4106 // can raise an error if class is undefined
4107 cgCallHelper(a,
4108 (TCA)StaticMethodCache::lookupIR,
4109 funcDestReg,
4110 kSyncPoint,
4111 ArgGroup().imm(ch) // Handle ch
4112 .immPtr(ne) // NamedEntity* np.second
4113 .immPtr(cls) // className
4114 .immPtr(method) // methodName
4115 );
4116 // recordInstrCall is done in cgCallHelper
4117 a.testq(funcDestReg, funcDestReg);
4118 a.loadq(rVmTl[ch + offsetof_cls], classDestReg);
4119 // if StaticMethodCache::lookupIR() returned NULL, jmp to label
4120 emitFwdJcc(a, CC_Z, label);
4121 });
4122 }
4123
4124 /**
4125 * Helper to emit getting the value for ActRec's m_this/m_cls slot
4126 * from a This pointer depending on whether the callee method is
4127 * static or not.
4128 */
4129 void CodeGenerator::emitGetCtxFwdCallWithThis(PhysReg ctxReg,
4130 bool staticCallee) {
4131 if (staticCallee) {
4132 // Load (this->m_cls | 0x1) into ctxReg.
4133 m_as.loadq(ctxReg[ObjectData::getVMClassOffset()], ctxReg);
4134 m_as.orq(1, ctxReg);
4135 } else {
4136 // Just incref $this.
4137 emitIncRef(m_as, ctxReg);
4138 }
4139 }
4140
4141 /**
4142 * This method is similar to emitGetCtxFwdCallWithThis above, but
4143 * whether or not the callee is a static method is unknown at JIT
4144 * time, and that is determined dynamically by looking up into the
4145 * StaticMethodFCache.
4146 */
4147 void CodeGenerator::emitGetCtxFwdCallWithThisDyn(PhysReg destCtxReg,
4148 PhysReg thisReg,
4149 CacheHandle& ch) {
4150 Label NonStaticCall, End;
4151
4152 // thisReg is holding $this. Should we pass it to the callee?
4153 m_as.cmpl(1, rVmTl[ch + offsetof(StaticMethodFCache, m_static)]);
4154 m_as.jcc8(CC_NE, NonStaticCall);
4155 // If calling a static method...
4156 {
4157 // Load (this->m_cls | 0x1) into destCtxReg
4158 m_as.loadq(thisReg[ObjectData::getVMClassOffset()], destCtxReg);
4159 m_as.orq(1, destCtxReg);
4160 m_as.jmp8(End);
4161 }
4162 // Else: calling non-static method
4163 {
4164 asm_label(m_as, NonStaticCall);
4165 emitMovRegReg(m_as, thisReg, destCtxReg);
4166 emitIncRef(m_as, destCtxReg);
4167 }
4168 asm_label(m_as, End);
4169 }
4170
4171 void CodeGenerator::cgGetCtxFwdCall(IRInstruction* inst) {
4172 PhysReg destCtxReg = inst->getDst()->getReg(0);
4173 SSATmp* srcCtxTmp = inst->getSrc(0);
4174 const Func* callee = inst->getSrc(1)->getValFunc();
4175 bool withThis = srcCtxTmp->isA(Type::Obj);
4176
4177 // Eagerly move src into the dest reg
4178 emitMovRegReg(m_as, srcCtxTmp->getReg(0), destCtxReg);
4179
4180 Label End;
4181 // If we don't know whether we have a This, we need to check dynamically
4182 if (!withThis) {
4183 m_as.testb(1, rbyte(destCtxReg));
4184 m_as.jcc8(CC_NZ, End);
4185 }
4186
4187 // If we have a This pointer in destCtxReg, then select either This
4188 // or its Class based on whether callee is static or not
4189 emitGetCtxFwdCallWithThis(destCtxReg, (callee->attrs() & AttrStatic));
4190
4191 asm_label(m_as, End);
4192 }
4193
4194 void CodeGenerator::cgLdClsMethodFCache(IRInstruction* inst) {
4195 PhysReg funcDestReg = inst->getDst()->getReg(0);
4196 PhysReg destCtxReg = inst->getDst()->getReg(1);
4197 const Class* cls = inst->getSrc(0)->getValClass();
4198 const StringData* methName = inst->getSrc(1)->getValStr();
4199 SSATmp* srcCtxTmp = inst->getSrc(2);
4200 PhysReg srcCtxReg = srcCtxTmp->getReg(0);
4201 Block* exitLabel = inst->getTaken();
4202 const StringData* clsName = cls->name();
4203 CacheHandle ch = StaticMethodFCache::alloc(clsName, methName,
4204 getContextName(getCurClass()));
4205
4206 assert(funcDestReg != InvalidReg && destCtxReg != InvalidReg);
4207 emitMovRegReg(m_as, srcCtxReg, destCtxReg);
4208 m_as.loadq(rVmTl[ch], funcDestReg);
4209 m_as.testq(funcDestReg, funcDestReg);
4210
4211 Label End;
4212
4213 // Handle case where method is not entered in the cache
4214 unlikelyIfBlock(CC_E, [&] (Asm& a) {
4215 const Func* (*lookup)(CacheHandle, const Class*, const StringData*) =
4216 StaticMethodFCache::lookupIR;
4217 // preserve destCtxReg across the call since it wouldn't be otherwise
4218 RegSet toSave = m_state.liveRegs[inst] | RegSet(destCtxReg);
4219 cgCallHelper(a, Transl::Call((TCA)lookup),
4220 funcDestReg, InvalidReg,
4221 kSyncPoint,
4222 ArgGroup().imm(ch)
4223 .immPtr(cls)
4224 .immPtr(methName),
4225 toSave);
4226 // If entry found in target cache, jump back to m_as.
4227 // Otherwise, bail to exit label
4228 a.testq(funcDestReg, funcDestReg);
4229 emitFwdJcc(a, CC_Z, exitLabel);
4230 });
4231
4232 auto t = srcCtxTmp->type();
4233 assert(!t.equals(Type::Cls));
4234 if (t.equals(Type::Cctx)) {
4235 return; // done: destCtxReg already has srcCtxReg
4236 } else if (t == Type::Obj) {
4237 // unconditionally run code produced by emitGetCtxFwdCallWithThisDyn below
4238 // break
4239 } else if (t == Type::Ctx) {
4240 // dynamically check if we have a This pointer and
4241 // call emitGetCtxFwdCallWithThisDyn below
4242 m_as.testb(1, rbyte(destCtxReg));
4243 m_as.jcc8(CC_NZ, End);
4244 } else {
4245 not_reached();
4246 }
4247
4248 // If we have a 'this' pointer ...
4249 emitGetCtxFwdCallWithThisDyn(destCtxReg, destCtxReg, ch);
4250
4251 asm_label(m_as, End);
4252 }
4253
4254 void CodeGenerator::cgLdClsPropAddrCached(IRInstruction* inst) {
4255 using namespace Transl::TargetCache;
4256 SSATmp* dst = inst->getDst();
4257 SSATmp* cls = inst->getSrc(0);
4258 SSATmp* propName = inst->getSrc(1);
4259 SSATmp* clsName = inst->getSrc(2);
4260 SSATmp* cxt = inst->getSrc(3);
4261 Block* target = inst->getTaken();
4262
4263 const StringData* propNameString = propName->getValStr();
4264 const StringData* clsNameString = clsName->getValStr();
4265
4266 string sds(Util::toLower(clsNameString->data()) + ":" +
4267 string(propNameString->data(), propNameString->size()));
4268 StackStringData sd(sds.c_str(), sds.size(), AttachLiteral);
4269 CacheHandle ch = SPropCache::alloc(&sd);
4270
4271 auto dstReg = dst->getReg();
4272 // Cls is live in the slow path call to lookupIR, so we have to be
4273 // careful not to clobber it before the branch to slow path. So
4274 // use the scratch register as a temporary destination if cls is
4275 // assigned the same register as the dst register.
4276 auto tmpReg = dstReg;
4277 if (dstReg == InvalidReg || dstReg == cls->getReg()) {
4278 tmpReg = PhysReg(rScratch);
4279 }
4280
4281 // Could be optimized to cmp against zero when !label && dstReg == InvalidReg
4282 m_as.loadq(rVmTl[ch], tmpReg);
4283 m_as.testq(tmpReg, tmpReg);
4284 unlikelyIfBlock(CC_E, [&] (Asm& a) {
4285 cgCallHelper(a,
4286 target ? (TCA)SPropCache::lookupIR<false>
4287 : (TCA)SPropCache::lookupIR<true>, // raise on error
4288 tmpReg,
4289 kSyncPoint, // could re-enter to initialize properties
4290 ArgGroup().imm(ch).ssa(cls).ssa(propName).ssa(cxt));
4291 if (target) {
4292 a.testq(tmpReg, tmpReg);
4293 emitFwdJcc(a, CC_Z, target);
4294 }
4295 });
4296 if (dstReg != InvalidReg) {
4297 emitMovRegReg(m_as, tmpReg, dstReg);
4298 }
4299 }
4300
4301 void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) {
4302 SSATmp* dst = inst->getDst();
4303 SSATmp* cls = inst->getSrc(0);
4304 SSATmp* prop = inst->getSrc(1);
4305 SSATmp* cxt = inst->getSrc(2);
4306 Block* target = inst->getTaken();
4307 auto dstReg = dst->getReg();
4308 if (dstReg == InvalidReg && target) {
4309 // result is unused but this instruction was not eliminated
4310 // because its essential
4311 dstReg = rScratch;
4312 }
4313 cgCallHelper(m_as,
4314 target ? (TCA)SPropCache::lookupSProp<false>
4315 : (TCA)SPropCache::lookupSProp<true>, // raise on error
4316 dstReg,
4317 kSyncPoint, // could re-enter to initialize properties
4318 ArgGroup().ssa(cls).ssa(prop).ssa(cxt));
4319 if (target) {
4320 m_as.testq(dstReg, dstReg);
4321 emitFwdJcc(m_as, CC_Z, target);
4322 }
4323 }
4324
4325 TargetCache::CacheHandle CodeGenerator::cgLdClsCachedCommon(
4326 IRInstruction* inst) {
4327 SSATmp* dst = inst->getDst();
4328 const StringData* className = inst->getSrc(0)->getValStr();
4329 auto ch = TargetCache::allocKnownClass(className);
4330 auto dstReg = dst->getReg();
4331 if (dstReg == InvalidReg) {
4332 m_as. cmpq (0, rVmTl[ch]);
4333 } else {
4334 m_as. loadq (rVmTl[ch], dstReg);
4335 m_as. testq (dstReg, dstReg);
4336 }
4337
4338 return ch;
4339 }
4340
4341 void CodeGenerator::cgLdClsCached(IRInstruction* inst) {
4342 auto ch = cgLdClsCachedCommon(inst);
4343 unlikelyIfBlock(CC_E, [&] (Asm& a) {
4344 // Passing only two arguments to lookupKnownClass, since the
4345 // third is ignored in the checkOnly==false case.
4346 cgCallHelper(a,
4347 (TCA)TargetCache::lookupKnownClass<false>,
4348 inst->getDst(),
4349 kSyncPoint,
4350 ArgGroup().addr(rVmTl, intptr_t(ch)).ssas(inst, 0));
4351 });
4352 }
4353
4354 void CodeGenerator::cgLdClsCachedSafe(IRInstruction* inst) {
4355 cgLdClsCachedCommon(inst);
4356 if (Block* taken = inst->getTaken()) {
4357 emitFwdJcc(CC_Z, taken);
4358 }
4359 }
4360
4361 void CodeGenerator::cgLdCls(IRInstruction* inst) {
4362 SSATmp* dst = inst->getDst();
4363 SSATmp* className = inst->getSrc(0);
4364
4365 CacheHandle ch = ClassCache::alloc();
4366 cgCallHelper(m_as, (TCA)ClassCache::lookup, dst, kSyncPoint,
4367 ArgGroup().imm(ch).ssa(className));
4368 }
4369
4370 static StringData* fullConstName(SSATmp* cls, SSATmp* cnsName) {
4371 return StringData::GetStaticString(
4372 Util::toLower(cls->getValStr()->data()) + "::" +
4373 cnsName->getValStr()->data());
4374 }
4375
4376 void CodeGenerator::cgLdClsCns(IRInstruction* inst) {
4377 SSATmp* cnsName = inst->getSrc(0);
4378 SSATmp* cls = inst->getSrc(1);
4379
4380 StringData* fullName = fullConstName(cls, cnsName);
4381 TargetCache::CacheHandle ch = TargetCache::allocClassConstant(fullName);
4382 // note that we bail from the trace if the target cache entry is empty
4383 // for this class constant or if the type assertion fails.
4384 // TODO: handle the slow case helper call.
4385 cgLoad(rVmTl, ch, inst);
4386 }
4387
4388 void CodeGenerator::cgLookupClsCns(IRInstruction* inst) {
4389 SSATmp* cnsName = inst->getSrc(0);
4390 SSATmp* cls = inst->getSrc(1);
4391
4392 assert(inst->getTypeParam() == Type::Cell);
4393 assert(cnsName->isConst() && cnsName->type() == Type::StaticStr);
4394 assert(cls->isConst() && cls->type() == Type::StaticStr);
4395
4396 StringData* fullName = fullConstName(cls, cnsName);
4397 TargetCache::CacheHandle ch = TargetCache::allocClassConstant(fullName);
4398
4399 ArgGroup args;
4400 args.addr(rVmTl, ch)
4401 .immPtr(Unit::GetNamedEntity(cls->getValStr()))
4402 .immPtr(cls->getValStr())
4403 .immPtr(cnsName->getValStr());
4404
4405 cgCallHelper(m_as, TCA(TargetCache::lookupClassConstantTv),
4406 inst->getDst(), kSyncPoint, args, DestType::TV);
4407 }
4408
4409 void CodeGenerator::cgLdCns(IRInstruction* inst) {
4410 const StringData* cnsName = inst->getSrc(0)->getValStr();
4411
4412 TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);
4413 // Has an unlikely branch to a LookupCns
4414 cgLoad(rVmTl, ch, inst);
4415 }
4416
4417 static TypedValue lookupCnsHelper(const TypedValue* tv, StringData* nm) {
4418 assert(tv->m_type == KindOfUninit);
4419 TypedValue *cns = nullptr;
4420 TypedValue c1;
4421 if (UNLIKELY(tv->m_data.pref != nullptr)) {
4422 ClassInfo::ConstantInfo* ci =
4423 (ClassInfo::ConstantInfo*)(void*)tv->m_data.pref;
4424 cns = const_cast<Variant&>(ci->getDeferredValue()).asTypedValue();
4425 tvReadCell(cns, &c1);
4426 } else {
4427 if (UNLIKELY(TargetCache::s_constants != nullptr)) {
4428 cns = TargetCache::s_constants->HphpArray::nvGet(nm);
4429 }
4430 if (!cns) {
4431 cns = Unit::loadCns(const_cast<StringData*>(nm));
4432 }
4433 if (UNLIKELY(!cns)) {
4434 raise_notice(Strings::UNDEFINED_CONSTANT, nm->data(), nm->data());
4435 c1.m_data.pstr = const_cast<StringData*>(nm);
4436 c1.m_type = KindOfStaticString;
4437 } else {
4438 c1.m_type = cns->m_type;
4439 c1.m_data = cns->m_data;
4440 }
4441 }
4442 return c1;
4443 }
4444
4445 void CodeGenerator::cgLookupCns(IRInstruction* inst) {
4446 SSATmp* cnsNameTmp = inst->getSrc(0);
4447
4448 assert(inst->getTypeParam() == Type::Cell);
4449 assert(cnsNameTmp->isConst() && cnsNameTmp->type() == Type::StaticStr);
4450
4451 const StringData* cnsName = cnsNameTmp->getValStr();
4452 TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);
4453
4454 ArgGroup args;
4455 args.addr(rVmTl, ch)
4456 .immPtr(cnsName);
4457
4458 cgCallHelper(m_as, TCA(lookupCnsHelper),
4459 inst->getDst(), kSyncPoint, args, DestType::TV);
4460 }
4461
4462 HOT_FUNC_VM
4463 static inline int64_t ak_exist_string_helper(StringData* key, ArrayData* arr) {
4464 int64_t n;
4465 if (key->isStrictlyInteger(n)) {
4466 return arr->exists(n);
4467 }
4468 return arr->exists(StrNR(key));
4469 }
4470
4471 HOT_FUNC_VM
4472 static int64_t ak_exist_string(StringData* key, ArrayData* arr) {
4473 int64_t res = ak_exist_string_helper(key, arr);
4474 return res;
4475 }
4476
4477 HOT_FUNC_VM
4478 static int64_t ak_exist_int(int64_t key, ArrayData* arr) {
4479 bool res = arr->exists(key);
4480 return res;
4481 }
4482
4483 HOT_FUNC_VM
4484 static int64_t ak_exist_string_obj(StringData* key, ObjectData* obj) {
4485 if (obj->isCollection()) {
4486 return collectionOffsetContains(obj, key);
4487 }
4488 CArrRef arr = obj->o_toArray();
4489 int64_t res = ak_exist_string_helper(key, arr.get());
4490 return res;
4491 }
4492
4493 HOT_FUNC_VM
4494 static int64_t ak_exist_int_obj(int64_t key, ObjectData* obj) {
4495 if (obj->isCollection()) {
4496 return collectionOffsetContains(obj, key);
4497 }
4498 CArrRef arr = obj->o_toArray();
4499 bool res = arr.get()->exists(key);
4500 return res;
4501 }
4502
4503 void CodeGenerator::cgAKExists(IRInstruction* inst) {
4504 SSATmp* arr = inst->getSrc(0);
4505 SSATmp* key = inst->getSrc(1);
4506
4507 if (key->type().isNull()) {
4508 if (arr->isA(Type::Arr)) {
4509 cgCallHelper(m_as,
4510 (TCA)ak_exist_string,
4511 inst->getDst(),
4512 kNoSyncPoint,
4513 ArgGroup().immPtr(empty_string.get()).ssa(arr));
4514 } else {
4515 m_as.mov_imm64_reg(0, inst->getDst()->getReg());
4516 }
4517 return;
4518 }
4519
4520 TCA helper_func =
4521 arr->isA(Type::Obj)
4522 ? (key->isA(Type::Int) ? (TCA)ak_exist_int_obj : (TCA)ak_exist_string_obj)
4523 : (key->isA(Type::Int) ? (TCA)ak_exist_int : (TCA)ak_exist_string);
4524
4525 cgCallHelper(m_as,
4526 helper_func,
4527 inst->getDst(),
4528 kNoSyncPoint,
4529 ArgGroup().ssa(key).ssa(arr));
4530 }
4531
4532 HOT_FUNC_VM static TypedValue* ldGblAddrHelper(StringData* name) {
4533 return g_vmContext->m_globalVarEnv->lookup(name);
4534 }
4535
4536 HOT_FUNC_VM static TypedValue* ldGblAddrDefHelper(StringData* name) {
4537 TypedValue* r = g_vmContext->m_globalVarEnv->lookupAdd(name);
4538 decRefStr(name);
4539 return r;
4540 }
4541
4542 void CodeGenerator::cgLdGblAddr(IRInstruction* inst) {
4543 auto dstReg = inst->getDst()->getReg();
4544 cgCallHelper(m_as, (TCA)ldGblAddrHelper, dstReg, kNoSyncPoint,
4545 ArgGroup().ssa(inst->getSrc(0)));
4546 m_as.testq(dstReg, dstReg);
4547 emitFwdJcc(CC_Z, inst->getTaken());
4548 }
4549
4550 void CodeGenerator::cgLdGblAddrDef(IRInstruction* inst) {
4551 cgCallHelper(m_as, (TCA)ldGblAddrDefHelper, inst->getDst(), kNoSyncPoint,
4552 ArgGroup().ssa(inst->getSrc(0)));
4553 }
4554
4555 void CodeGenerator::cgJmpZeroHelper(IRInstruction* inst,
4556 ConditionCode cc) {
4557 SSATmp* src = inst->getSrc(0);
4558
4559 auto srcReg = src->getReg();
4560 if (src->isConst()) {
4561 bool valIsZero = src->getValRawInt() == 0;
4562 if ((cc == CC_Z && valIsZero) ||
4563 (cc == CC_NZ && !valIsZero)) {
4564 // assert(false) here after new simplifier pass, t2019643
4565 // For now, materialize the test condition and use a Jcc
4566 m_as.xor_reg64_reg64(rScratch, rScratch);
4567 m_as.test_reg64_reg64(rScratch, rScratch);
4568 cc = CC_Z;
4569 // Update the instr opcode since cgExitTrace uses it
4570 // to determine correct cc for service request.
4571 inst->setOpcode(JmpZero);
4572 } else {
4573 // Fall through to next bytecode, disable DirectJmp
4574 inst->setTCA(kIRDirectJmpInactive);
4575 return;
4576 }
4577 } else {
4578 if (src->type() == Type::Bool) {
4579 m_as.testb(Reg8(int(srcReg)), Reg8(int(srcReg)));
4580 } else {
4581 m_as.test_reg64_reg64(srcReg, srcReg);
4582 }
4583 }
4584
4585 emitJccDirectExit(inst, cc);
4586 }
4587
4588 void CodeGenerator::cgJmpZero(IRInstruction* inst) {
4589 cgJmpZeroHelper(inst, CC_Z);
4590 }
4591
4592 void CodeGenerator::cgJmpNZero(IRInstruction* inst) {
4593 cgJmpZeroHelper(inst, CC_NZ);
4594 }
4595
4596 void CodeGenerator::cgJmp_(IRInstruction* inst) {
4597 assert(inst->getNumSrcs() == inst->getTaken()->getLabel()->getNumDsts());
4598 if (unsigned n = inst->getNumSrcs()) {
4599 // Parallel-copy sources to the label's destination registers.
4600 // TODO: t2040286: this only works if all destinations fit in registers.
4601 SrcRange srcs = inst->getSrcs();
4602 DstRange dsts = inst->getTaken()->getLabel()->getDsts();
4603 ArgGroup args;
4604 for (unsigned i = 0, j = 0; i < n; i++) {
4605 assert(srcs[i]->type().subtypeOf(dsts[i].type()));
4606 SSATmp *dst = &dsts[i], *src = srcs[i];
4607 if (dst->getReg(0) == InvalidReg) continue; // dst is unused.
4608 // first dst register
4609 args.ssa(src);
4610 args[j++].setDstReg(dst->getReg(0));
4611 // second dst register, if any
4612 if (dst->numNeededRegs() == 2) {
4613 if (src->numNeededRegs() < 2) {
4614 // src has known data type, but dst doesn't - pass immediate type
4615 assert(src->type().isKnownDataType());
4616 args.imm(src->type().toDataType());
4617 } else {
4618 // pass src's second register
4619 assert(src->getReg(1) != InvalidReg);
4620 args.reg(src->getReg(1));
4621 }
4622 args[j++].setDstReg(dst->getReg(1));
4623 }
4624 }
4625 shuffleArgs(m_as, args);
4626 }
4627 if (!m_state.noTerminalJmp_) {
4628 emitFwdJmp(inst->getTaken());
4629 }
4630 }
4631
4632 void CodeGenerator::cgJmpIndirect(IRInstruction* inst) {
4633 m_as.jmp(inst->getSrc(0)->getReg());
4634 }
4635
4636 void CodeGenerator::cgCheckInit(IRInstruction* inst) {
4637 Block* label = inst->getTaken();
4638 assert(label);
4639 SSATmp* src = inst->getSrc(0);
4640
4641 if (src->type().isInit()) return;
4642
4643 auto typeReg = src->getReg(1);
4644 assert(typeReg != InvalidReg);
4645
4646 static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0");
4647 m_as.testb (rbyte(typeReg), rbyte(typeReg));
4648 emitFwdJcc(CC_Z, label);
4649 }
4650
4651 void CodeGenerator::cgCheckInitMem(IRInstruction* inst) {
4652 Block* label = inst->getTaken();
4653 assert(label);
4654 SSATmp* base = inst->getSrc(0);
4655 int64_t offset = inst->getSrc(1)->getValInt();
4656 Type t = base->type().deref();
4657 if (t.isInit()) return;
4658 emitCmpTVType(m_as, KindOfUninit, base->getReg()[offset + TVOFF(m_type)]);
4659 emitFwdJcc(CC_Z, label);
4660 }
4661
4662 void CodeGenerator::cgExitWhenSurprised(IRInstruction* inst) {
4663 Block* label = inst->getTaken();
4664 m_tx64->emitTestSurpriseFlags(m_as);
4665 emitFwdJcc(CC_NZ, label);
4666 }
4667
4668 void CodeGenerator::cgExitOnVarEnv(IRInstruction* inst) {
4669 SSATmp* fp = inst->getSrc(0);
4670 Block* label = inst->getTaken();
4671
4672 assert(!(fp->isConst()));
4673
4674 auto fpReg = fp->getReg();
4675 m_as. cmpq (0, fpReg[AROFF(m_varEnv)]);
4676 emitFwdJcc(CC_NE, label);
4677 }
4678
4679 void CodeGenerator::cgReleaseVVOrExit(IRInstruction* inst) {
4680 auto* const label = inst->getTaken();
4681 auto const rFp = inst->getSrc(0)->getReg();
4682
4683 m_as. cmpq (0, rFp[AROFF(m_varEnv)]);
4684 unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
4685 a. testl (ActRec::kExtraArgsBit, rFp[AROFF(m_varEnv)]);
4686 emitFwdJcc(a, CC_Z, label);
4687 cgCallHelper(
4688 a,
4689 TCA(static_cast<void (*)(ActRec*)>(ExtraArgs::deallocate)),
4690 nullptr,
4691 kSyncPoint,
4692 ArgGroup().reg(rFp),
4693 DestType::None
4694 );
4695 });
4696 }
4697
4698 void CodeGenerator::cgBoxPtr(IRInstruction* inst) {
4699 SSATmp* dst = inst->getDst();
4700 SSATmp* addr = inst->getSrc(0);
4701 auto base = addr->getReg();
4702 auto dstReg = dst->getReg();
4703 emitMovRegReg(m_as, base, dstReg);
4704 ConditionCode cc = emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)], true);
4705 ifThen(m_as, cc, [&] {
4706 cgCallHelper(m_as, (TCA)tvBox, dstReg, kNoSyncPoint, ArgGroup().ssa(addr));
4707 });
4708 }
4709
4710 void CodeGenerator::cgDefCns(IRInstruction* inst) {
4711 UNUSED SSATmp* dst = inst->getDst();
4712 UNUSED SSATmp* cnsName = inst->getSrc(0);
4713 UNUSED SSATmp* val = inst->getSrc(1);
4714 using namespace TargetCache;
4715 CG_PUNT(DefCns);
4716 }
4717
4718 // TODO: Kill this #2031980
4719 static StringData* concat_value(TypedValue tv1, TypedValue tv2) {
4720 return concat_tv(tv1.m_type, tv1.m_data.num, tv2.m_type, tv2.m_data.num);
4721 }
4722
4723 void CodeGenerator::cgConcat(IRInstruction* inst) {
4724 SSATmp* dst = inst->getDst();
4725 SSATmp* tl = inst->getSrc(0);
4726 SSATmp* tr = inst->getSrc(1);
4727
4728 Type lType = tl->type();
4729 Type rType = tr->type();
4730 // We have specialized helpers for concatenating two strings, a
4731 // string and an int, and an int and a string.
4732 void* fptr = nullptr;
4733 if (lType.isString() && rType.isString()) {
4734 fptr = (void*)concat_ss;
4735 } else if (lType.isString() && rType == Type::Int) {
4736 fptr = (void*)concat_si;
4737 } else if (lType == Type::Int && rType.isString()) {
4738 fptr = (void*)concat_is;
4739 }
4740 if (fptr) {
4741 cgCallHelper(m_as, (TCA)fptr, dst, kNoSyncPoint,
4742 ArgGroup().ssa(tl)
4743 .ssa(tr));
4744 } else {
4745 if (lType.subtypeOf(Type::Obj) || lType.needsReg() ||
4746 rType.subtypeOf(Type::Obj) || rType.needsReg()) {
4747 CG_PUNT(cgConcat);
4748 }
4749 cgCallHelper(m_as, (TCA)concat_value, dst, kNoSyncPoint,
4750 ArgGroup().typedValue(tl).typedValue(tr));
4751 }
4752 }
4753
4754 void CodeGenerator::cgInterpOne(IRInstruction* inst) {
4755 SSATmp* fp = inst->getSrc(0);
4756 SSATmp* sp = inst->getSrc(1);
4757 SSATmp* pcOffTmp = inst->getSrc(2);
4758 SSATmp* spAdjustmentTmp = inst->getSrc(3);
4759 Type resultType = inst->getTypeParam();
4760 int64_t pcOff = pcOffTmp->getValInt();
4761
4762 auto opc = *(getCurFunc()->unit()->at(pcOff));
4763 void* interpOneHelper = interpOneEntryPoints[opc];
4764
4765 auto dstReg = InvalidReg;
4766 cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
4767 ArgGroup().ssa(fp).ssa(sp).imm(pcOff));
4768
4769 auto newSpReg = inst->getDst()->getReg();
4770 assert(newSpReg == sp->getReg());
4771
4772 int64_t spAdjustBytes = cellsToBytes(spAdjustmentTmp->getValInt());
4773 if (spAdjustBytes != 0) {
4774 m_as.addq(spAdjustBytes, newSpReg);
4775 }
4776 }
4777
4778 void CodeGenerator::cgInterpOneCF(IRInstruction* inst) {
4779 SSATmp* fp = inst->getSrc(0);
4780 SSATmp* sp = inst->getSrc(1);
4781 int64_t pcOff = inst->getSrc(2)->getValInt();
4782
4783 auto opc = *(getCurFunc()->unit()->at(pcOff));
4784 void* interpOneHelper = interpOneEntryPoints[opc];
4785
4786 auto dstReg = InvalidReg;
4787 cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
4788 ArgGroup().ssa(fp).ssa(sp).imm(pcOff));
4789
4790 // The interpOne method returns a pointer to the current ExecutionContext
4791 // in rax. Use it read the 'm_fp' and 'm_stack.m_top' fields into the
4792 // rVmFp and rVmSp registers.
4793 m_as.loadq(rax[offsetof(VMExecutionContext, m_fp)], rVmFp);
4794 m_as.loadq(rax[offsetof(VMExecutionContext, m_stack) +
4795 Stack::topOfStackOffset()], rVmSp);
4796
4797 m_tx64->emitServiceReq(TranslatorX64::SRFlags::SREmitInA, REQ_RESUME, 0ull);
4798 }
4799
4800 void CodeGenerator::cgDefFunc(IRInstruction* inst) {
4801 SSATmp* dst = inst->getDst();
4802 SSATmp* func = inst->getSrc(0);
4803 cgCallHelper(m_as, (TCA)defFuncHelper, dst, kSyncPoint,
4804 ArgGroup().ssa(func), DestType::None);
4805 }
4806
4807 void CodeGenerator::cgFillContThis(IRInstruction* inst) {
4808 SSATmp* cont = inst->getSrc(0);
4809 auto baseReg = inst->getSrc(1)->getReg();
4810 int64_t offset = inst->getSrc(2)->getValInt();
4811 auto scratch = rScratch;
4812 auto contReg = cont->getReg();
4813
4814 m_as.loadq(contReg[CONTOFF(m_obj)], scratch);
4815 m_as.testq(scratch, scratch);
4816 ifThen(m_as, CC_NZ, [&] {
4817 m_as.addl(1, scratch[FAST_REFCOUNT_OFFSET]);
4818 m_as.storeq(scratch, baseReg[offset + TVOFF(m_data)]);
4819 emitStoreTVType(m_as, KindOfObject, baseReg[offset + TVOFF(m_type)]);
4820 });
4821 }
4822
4823 void CodeGenerator::cgContEnter(IRInstruction* inst) {
4824 auto contAR = inst->getSrc(0);
4825 auto addr = inst->getSrc(1);
4826 auto returnOff = inst->getSrc(2);
4827 auto curFp = inst->getSrc(3)->getReg();
4828 auto contARReg = contAR->getReg();
4829
4830 m_as. storel (returnOff->getValInt(), contARReg[AROFF(m_soff)]);
4831 m_as. storeq (curFp, contARReg[AROFF(m_savedRbp)]);
4832 m_as. movq (contARReg, rStashedAR);
4833
4834 m_as. call (addr->getReg());
4835 }
4836
4837 void CodeGenerator::emitContVarEnvHelperCall(SSATmp* fp, TCA helper) {
4838 auto scratch = rScratch;
4839
4840 m_as. loadq (fp->getReg()[AROFF(m_varEnv)], scratch);
4841 m_as. testq (scratch, scratch);
4842 unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
4843 cgCallHelper(a, helper, InvalidReg, kNoSyncPoint,
4844 ArgGroup().ssa(fp));
4845 });
4846 }
4847
4848 void CodeGenerator::cgUnlinkContVarEnv(IRInstruction* inst) {
4849 emitContVarEnvHelperCall(
4850 inst->getSrc(0),
4851 (TCA)VMExecutionContext::packContVarEnvLinkage);
4852 }
4853
4854 void CodeGenerator::cgLinkContVarEnv(IRInstruction* inst) {
4855 emitContVarEnvHelperCall(
4856 inst->getSrc(0),
4857 (TCA)VMExecutionContext::unpackContVarEnvLinkage);
4858 }
4859
4860 void CodeGenerator::cgContRaiseCheck(IRInstruction* inst) {
4861 SSATmp* cont = inst->getSrc(0);
4862 m_as.test_imm32_disp_reg32(0x1, CONTOFF(m_should_throw),
4863 cont->getReg());
4864 emitFwdJcc(CC_NZ, inst->getTaken());
4865 }
4866
4867 void CodeGenerator::cgContPreNext(IRInstruction* inst) {
4868 auto contReg = inst->getSrc(0)->getReg();
4869
4870 const Offset doneOffset = CONTOFF(m_done);
4871 static_assert((doneOffset + 1) == CONTOFF(m_running),
4872 "m_done should immediately precede m_running");
4873 // Check m_done and m_running at the same time
4874 m_as.test_imm32_disp_reg32(0x0101, doneOffset, contReg);
4875 emitFwdJcc(CC_NZ, inst->getTaken());
4876
4877 // ++m_index
4878 m_as.add_imm64_disp_reg64(0x1, CONTOFF(m_index), contReg);
4879 // m_running = true
4880 m_as.store_imm8_disp_reg(0x1, CONTOFF(m_running), contReg);
4881 }
4882
4883 void CodeGenerator::cgContStartedCheck(IRInstruction* inst) {
4884 m_as.cmp_imm64_disp_reg64(0, CONTOFF(m_index),
4885 inst->getSrc(0)->getReg());
4886 emitFwdJcc(CC_L, inst->getTaken());
4887 }
4888
4889 void CodeGenerator::cgIterNextK(IRInstruction* inst) {
4890 cgIterNextCommon(inst, true);
4891 }
4892
4893 void CodeGenerator::cgIterNext(IRInstruction* inst) {
4894 cgIterNextCommon(inst, false);
4895 }
4896
4897 void CodeGenerator::cgIterNextCommon(IRInstruction* inst, bool isNextK) {
4898 PhysReg fpReg = inst->getSrc(0)->getReg();
4899 ArgGroup args;
4900 args.addr(fpReg, getIterOffset(inst->getSrc(1)))
4901 .addr(fpReg, getLocalOffset(inst->getSrc(2)));
4902 if (isNextK) {
4903 args.addr(fpReg, getLocalOffset(inst->getSrc(3)));
4904 }
4905 TCA helperAddr = isNextK ? (TCA)iter_next_key : (TCA)iter_next;
4906 cgCallHelper(m_as, helperAddr, inst->getDst(), kSyncPoint, args);
4907 }
4908
4909 void CodeGenerator::cgIterInit(IRInstruction* inst) {
4910 cgIterInitCommon(inst, false);
4911 }
4912
4913 void iterFreeHelper(Iter* iter) {
4914 iter->free();
4915 }
4916
4917 void CodeGenerator::cgIterFree(IRInstruction* inst) {
4918 PhysReg fpReg = inst->getSrc(0)->getReg();
4919 int64_t offset = getIterOffset(inst->getSrc(1));
4920 cgCallHelper(m_as, (TCA)iterFreeHelper, InvalidReg, kSyncPoint,
4921 ArgGroup().addr(fpReg, offset));
4922 }
4923
4924 void CodeGenerator::cgIterInitK(IRInstruction* inst) {
4925 cgIterInitCommon(inst, true);
4926 }
4927
4928 void CodeGenerator::cgIterInitCommon(IRInstruction* inst, bool isInitK) {
4929 PhysReg fpReg = inst->getSrc(1)->getReg();
4930 int64_t iterOffset = getIterOffset(inst->getSrc(2));
4931 int64_t valLocalOffset = getLocalOffset(inst->getSrc(3));
4932 SSATmp* src = inst->getSrc(0);
4933 ArgGroup args;
4934 args.addr(fpReg, iterOffset).ssa(src);
4935 if (src->isArray()) {
4936 args.addr(fpReg, valLocalOffset);
4937 if (isInitK) {
4938 args.addr(fpReg, getLocalOffset(inst->getSrc(4)));
4939 }
4940 TCA helperAddr = isInitK ? (TCA)new_iter_array_key : (TCA)new_iter_array;
4941 cgCallHelper(m_as, helperAddr, inst->getDst(), kSyncPoint, args);
4942 } else {
4943 assert(src->type() == Type::Obj);
4944 args.imm(uintptr_t(getCurClass())).addr(fpReg, valLocalOffset);
4945 if (isInitK) {
4946 args.addr(fpReg, getLocalOffset(inst->getSrc(4)));
4947 } else {
4948 args.imm(0);
4949 }
4950 // new_iter_object decrefs its src object if it propagates an
4951 // exception out, so we use kSyncPointAdjustOne, which adjusts the
4952 // stack pointer by 1 stack element on an unwind, skipping over
4953 // the src object.
4954 cgCallHelper(m_as, (TCA)new_iter_object, inst->getDst(),
4955 kSyncPointAdjustOne, args);
4956 }
4957 }
4958
4959 void CodeGenerator::cgIncStat(IRInstruction *inst) {
4960 Stats::emitInc(m_as,
4961 Stats::StatCounter(inst->getSrc(0)->getValInt()),
4962 inst->getSrc(1)->getValInt(),
4963 Transl::CC_None,
4964 inst->getSrc(2)->getValBool());
4965 }
4966
4967 void CodeGenerator::cgIncTransCounter(IRInstruction* inst) {
4968 m_tx64->emitTransCounterInc(m_as);
4969 }
4970
4971 void CodeGenerator::cgDbgAssertRefCount(IRInstruction* inst) {
4972 emitAssertRefCount(m_as, inst->getSrc(0)->getReg());
4973 }
4974
4975 void traceCallback(ActRec* fp, Cell* sp, int64_t pcOff, void* rip) {
4976 if (HPHP::Trace::moduleEnabled(HPHP::Trace::hhirTracelets)) {
4977 FTRACE(0, "{} {} {}\n", fp->m_func->fullName()->data(), pcOff, rip);
4978 }
4979 checkFrame(fp, sp, /*checkLocals*/true);
4980 }
4981
4982 void CodeGenerator::cgVerifyParamCls(IRInstruction* inst) {
4983 SSATmp* objClass = inst->getSrc(0);
4984 assert(!objClass->isConst());
4985 auto objClassReg = objClass->getReg();
4986 SSATmp* constraint = inst->getSrc(1);
4987
4988 if (constraint->isConst()) {
4989 m_as. cmpq(constraint->getValClass(), objClassReg);
4990 } else {
4991 m_as. cmpq(constraint->getReg(), objClassReg);
4992 }
4993
4994 // The native call for this instruction is the slow path that does
4995 // proper subtype checking. The comparison above is just to
4996 // short-circuit the overhead when the Classes are an exact match.
4997 ifThen(m_as, CC_NE, [&]{ cgCallNative(inst); });
4998 }
4999
5000 void CodeGenerator::emitTraceCall(CodeGenerator::Asm& as, int64_t pcOff,
5001 Transl::TranslatorX64* tx64) {
5002 // call to a trace function
5003 as.mov_imm64_reg((int64_t)as.code.frontier, reg::rcx);
5004 as.mov_reg64_reg64(rVmFp, reg::rdi);
5005 as.mov_reg64_reg64(rVmSp, reg::rsi);
5006 as.mov_imm64_reg(pcOff, reg::rdx);
5007 // do the call; may use a trampoline
5008 tx64->emitCall(as, (TCA)traceCallback);
5009 }
5010
5011 static void patchJumps(Asm& as, CodegenState& state, Block* block) {
5012 void* list = state.patches[block];
5013 Address labelAddr = as.code.frontier;
5014 while (list) {
5015 int32_t* toPatch = (int32_t*)list;
5016 int32_t diffToNext = *toPatch;
5017 ssize_t diff = labelAddr - ((Address)list + sizeof(int32_t));
5018 *toPatch = safe_cast<int32_t>(diff); // patch the jump address
5019 if (diffToNext == 0) break;
5020 void* next = (TCA)list - diffToNext;
5021 list = next;
5022 }
5023 }
5024
5025 void CodeGenerator::cgBlock(Block* block, vector<TransBCMapping>* bcMap) {
5026 for (IRInstruction& instr : *block) {
5027 IRInstruction* inst = &instr;
5028 if (inst->op() == Marker) {
5029 m_state.lastMarker = inst->getExtra<Marker>();
5030 if (m_tx64 && m_tx64->isTransDBEnabled() && bcMap) {
5031 bcMap->push_back((TransBCMapping){Offset(m_state.lastMarker->bcOff),
5032 m_as.code.frontier,
5033 m_astubs.code.frontier});
5034 }
5035 }
5036 m_curInst = inst;
5037 auto nuller = folly::makeGuard([&]{ m_curInst = nullptr; });
5038 auto* addr = cgInst(inst);
5039 if (m_state.asmInfo && addr) {
5040 m_state.asmInfo->instRanges[inst] = TcaRange(addr, m_as.code.frontier);
5041 m_state.asmInfo->asmRanges[block] =
5042 TcaRange(m_state.asmInfo->asmRanges[block].start(), m_as.code.frontier);
5043 }
5044 }
5045 }
5046
5047 void cgTrace(Trace* trace, Asm& amain, Asm& astubs, Transl::TranslatorX64* tx64,
5048 vector<TransBCMapping>* bcMap, CodegenState& state) {
5049 state.lastMarker = nullptr;
5050 if (RuntimeOption::EvalHHIRGenerateAsserts && trace->isMain()) {
5051 CodeGenerator::emitTraceCall(amain, trace->getBcOff(), tx64);
5052 }
5053 auto chooseAs = [&](Block* b) {
5054 return b->getHint() != Block::Unlikely ? &amain : &astubs;
5055 };
5056 auto& blocks = trace->getBlocks();
5057 for (auto it = blocks.begin(), end = blocks.end(); it != end;) {
5058 Block* block = *it; ++it;
5059 Asm* as = chooseAs(block);
5060 TCA asmStart = as->code.frontier;
5061 TCA astubsStart = astubs.code.frontier;
5062 patchJumps(*as, state, block);
5063
5064 // Grab the next block that will go into this assembler
5065 Block* nextThisAs = nullptr;
5066 for (auto next = it; next != end; ++next) {
5067 if (chooseAs(*next) == as) {
5068 nextThisAs = *next;
5069 break;
5070 }
5071 }
5072
5073 // If the block ends with a Jmp_ to the next block for this
5074 // assembler, it doesn't need to actually emit a jmp.
5075 IRInstruction* last = block->back();
5076 state.noTerminalJmp_ =
5077 last->op() == Jmp_ && last->getTaken() == nextThisAs;
5078
5079 CodeGenerator cg(trace, *as, astubs, tx64, state);
5080 if (state.asmInfo) {
5081 state.asmInfo->asmRanges[block] = TcaRange(asmStart, as->code.frontier);
5082 }
5083 cg.cgBlock(block, bcMap);
5084 Block* next = block->getNext();
5085 if (next && next != nextThisAs) {
5086 // if there's a fallthrough block and it's not the next thing
5087 // going into this assembler, then emit a jump to it.
5088 CodeGenerator::emitFwdJmp(*as, next, state);
5089 }
5090 if (state.asmInfo) {
5091 state.asmInfo->asmRanges[block] = TcaRange(asmStart, as->code.frontier);
5092 if (as != &astubs) {
5093 state.asmInfo->astubRanges[block] = TcaRange(astubsStart,
5094 astubs.code.frontier);
5095 }
5096 }
5097 }
5098 }
5099
5100 void CodeGenerator::print() const {
5101 JIT::print(std::cout, m_curTrace, m_state.asmInfo);
5102 }
5103
5104 /*
5105 * Compute and save registers that are live *across* each inst, not including
5106 * registers whose lifetimes end at inst, nor registers defined by inst.
5107 */
5108 LiveRegs computeLiveRegs(const IRFactory* factory, Block* start_block) {
5109 StateVector<Block, RegSet> liveMap(factory, RegSet());
5110 LiveRegs live_regs(factory, RegSet());
5111 postorderWalk(
5112 [&](Block* block) {
5113 RegSet& live = liveMap[block];
5114 if (Block* taken = block->getTaken()) live = liveMap[taken];
5115 if (Block* next = block->getNext()) live |= liveMap[next];
5116 for (auto it = block->end(); it != block->begin(); ) {
5117 IRInstruction& inst = *--it;
5118 for (const SSATmp& dst : inst.getDsts()) {
5119 live -= dst.getRegs();
5120 }
5121 live_regs[inst] = live;
5122 for (SSATmp* src : inst.getSrcs()) {
5123 live |= src->getRegs();
5124 }
5125 }
5126 },
5127 factory->numBlocks(),
5128 start_block
5129 );
5130 return live_regs;
5131 }
5132
5133 // select instructions for the trace and its exits
5134 void genCodeForTrace(Trace* trace,
5135 CodeGenerator::Asm& as,
5136 CodeGenerator::Asm& astubs,
5137 IRFactory* irFactory,
5138 vector<TransBCMapping>* bcMap,
5139 Transl::TranslatorX64* tx64,
5140 AsmInfo* asmInfo) {
5141 assert(trace->isMain());
5142 LiveRegs live_regs = computeLiveRegs(irFactory, trace->front());
5143 CodegenState state(irFactory, live_regs, asmInfo);
5144 cgTrace(trace, as, astubs, tx64, bcMap, state);
5145 for (Trace* exit : trace->getExitTraces()) {
5146 cgTrace(exit, astubs, astubs, tx64, nullptr, state);
5147 }
5148 }
5149
5150 }}}