Fix spilling bug
[hiphop-php.git] / hphp / runtime / vm / jit / translator-x64.h
blob2688d150490850bcfac5a1f3181bb00670127e8e
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
16 #ifndef incl_HPHP_RUNTIME_VM_TRANSLATOR_X64_H_
17 #define incl_HPHP_RUNTIME_VM_TRANSLATOR_X64_H_
19 #include <signal.h>
20 #include <memory>
21 #include <boost/noncopyable.hpp>
23 #include "hphp/runtime/vm/bytecode.h"
24 #include "hphp/runtime/vm/jit/translator.h"
25 #include "hphp/util/asm-x64.h"
26 #include "hphp/runtime/vm/jit/unwind-x64.h"
27 #include "tbb/concurrent_hash_map.h"
28 #include "hphp/util/ringbuffer.h"
29 #include "hphp/runtime/vm/debug/debug.h"
30 #include "hphp/runtime/vm/jit/abi-x64.h"
32 namespace HPHP { class ExecutionContext; }
34 namespace HPHP { namespace JIT {
35 class HhbcTranslator;
36 class IRFactory;
37 class CSEHash;
38 class TraceBuilder;
39 class CodeGenerator;
42 namespace HPHP { namespace Transl {
44 struct TraceletCounters {
45 uint64_t m_numEntered, m_numExecuted;
48 struct TraceletCountersVec {
49 int64_t m_size;
50 TraceletCounters *m_elms;
51 Mutex m_lock;
53 TraceletCountersVec() : m_size(0), m_elms(nullptr), m_lock() { }
56 struct FreeStubList {
57 struct StubNode {
58 StubNode* m_next;
59 uint64_t m_freed;
61 static const uint64_t kStubFree = 0;
62 StubNode* m_list;
63 FreeStubList() : m_list(nullptr) {}
64 TCA maybePop();
65 void push(TCA stub);
68 struct CppCall {
69 explicit CppCall(void *p) : m_kind(Direct), m_fptr(p) {}
70 explicit CppCall(int off) : m_kind(Virtual), m_offset(off) {}
71 CppCall(CppCall const&) = default;
73 bool isDirect() const { return m_kind == Direct; }
74 bool isVirtual() const { return m_kind == Virtual; }
76 const void* getAddress() const { return m_fptr; }
77 int getOffset() const { return m_offset; }
79 private:
80 enum { Direct, Virtual } m_kind;
81 union {
82 void* m_fptr;
83 int m_offset;
87 class TranslatorX64;
88 extern __thread TranslatorX64* tx64;
90 extern void* interpOneEntryPoints[];
92 extern "C" TCA funcBodyHelper(ActRec* fp);
94 struct TReqInfo;
95 struct Label;
97 static const int kNumFreeLocalsHelpers = 9;
99 typedef X64Assembler Asm;
101 constexpr size_t kJmpTargetAlign = 16;
102 constexpr size_t kNonFallthroughAlign = 64;
103 constexpr int kJmpLen = 5;
104 constexpr int kCallLen = 5;
105 constexpr int kJmpccLen = 6;
106 constexpr int kJmpImmBytes = 4;
107 constexpr int kJcc8Len = 3;
108 constexpr int kLeaRipLen = 7;
109 constexpr int kTestRegRegLen = 3;
110 constexpr int kTestImmRegLen = 5; // only for rax -- special encoding
111 // Cache alignment is required for mutable instructions to make sure
112 // mutations don't "tear" on remote cpus.
113 constexpr size_t kX64CacheLineSize = 64;
114 constexpr size_t kX64CacheLineMask = kX64CacheLineSize - 1;
116 enum class TestAndSmashFlags {
117 kAlignJccImmediate,
118 kAlignJcc,
119 kAlignJccAndJmp
121 void prepareForTestAndSmash(Asm&, int testBytes, TestAndSmashFlags flags);
122 void prepareForSmash(Asm&, int nBytes, int offset = 0);
123 bool isSmashable(Address frontier, int nBytes, int offset = 0);
125 class TranslatorX64 : public Translator
126 , boost::noncopyable {
127 friend class SrcRec; // so it can smash code.
128 friend class SrcDB; // For write lock and code invalidation.
129 friend class Tx64Reaper;
130 friend class HPHP::JIT::CodeGenerator;
132 typedef tbb::concurrent_hash_map<TCA, TCA> SignalStubMap;
133 typedef void (*sigaction_t)(int, siginfo_t*, void*);
135 typedef X64Assembler Asm;
137 class AHotSelector {
138 public:
139 AHotSelector(TranslatorX64* tx, bool hot) :
140 m_tx(tx), m_hot(hot &&
141 tx->ahot.code.base + tx->ahot.code.size -
142 tx->ahot.code.frontier > 8192 &&
143 tx->a.code.base != tx->ahot.code.base) {
144 if (m_hot) {
145 m_save = tx->a;
146 tx->a = tx->ahot;
149 ~AHotSelector() {
150 if (m_hot) {
151 m_tx->ahot = m_tx->a;
152 m_tx->a = m_save;
155 private:
156 TranslatorX64* m_tx;
157 Asm m_save;
158 bool m_hot;
161 Asm ahot;
162 Asm a;
163 Asm astubs;
164 Asm atrampolines;
165 PointerMap trampolineMap;
166 int m_numNativeTrampolines;
167 size_t m_trampolineSize; // size of each trampoline
169 SignalStubMap m_segvStubs;
170 sigaction_t m_segvChain;
171 TCA m_callToExit;
172 TCA m_retHelper;
173 TCA m_retInlHelper;
174 TCA m_genRetHelper;
175 TCA m_stackOverflowHelper;
176 TCA m_irPopRHelper;
177 TCA m_dtorGenericStub;
178 TCA m_dtorGenericStubRegs;
179 TCA m_dtorStubs[kDestrTableSize];
180 TCA m_defClsHelper;
181 TCA m_funcPrologueRedispatch;
183 TCA m_freeManyLocalsHelper;
184 TCA m_freeLocalsHelpers[kNumFreeLocalsHelpers];
186 DataBlock m_globalData;
188 // Data structures for HHIR-based translation
189 uint64_t m_numHHIRTrans;
191 virtual void traceCodeGen();
193 FixupMap m_fixupMap;
194 UnwindInfoHandle m_unwindRegistrar;
195 CatchTraceMap m_catchTraceMap;
196 std::vector<TransBCMapping> m_bcMap;
198 Debug::DebugInfo m_debugInfo;
200 private:
201 int64_t m_createdTime;
203 struct PendingFixup {
204 TCA m_tca;
205 Fixup m_fixup;
206 PendingFixup() { }
207 PendingFixup(TCA tca, Fixup fixup) :
208 m_tca(tca), m_fixup(fixup) { }
210 vector<PendingFixup> m_pendingFixups;
212 void drawCFG(std::ofstream& out) const;
213 static vector<PhysReg> x64TranslRegs();
215 Asm& getAsmFor(TCA addr) { return asmChoose(addr, a, ahot, astubs); }
216 void emitIncRef(X64Assembler &a, PhysReg base, DataType dtype);
217 void emitIncRef(PhysReg base, DataType);
218 void emitIncRefGenericRegSafe(PhysReg base, int disp, PhysReg tmp);
219 static CppCall getDtorCall(DataType type);
220 void emitCopy(PhysReg srcCell, int disp, PhysReg destCell);
222 void emitThisCheck(const NormalizedInstruction& i, PhysReg reg);
224 public:
225 void emitCall(Asm& a, TCA dest);
226 void emitCall(Asm& a, CppCall call);
227 TCA getCallArrayProlog(Func* func);
228 private:
230 void translateClassExistsImpl(const Tracelet& t,
231 const NormalizedInstruction& i,
232 Attr typeAttr);
233 void recordSyncPoint(Asm& a, Offset pcOff, Offset spOff);
234 void emitEagerSyncPoint(Asm& a, const Opcode* pc, const Offset spDiff);
235 void recordIndirectFixup(CTCA addr, int dwordsPushed);
236 void emitStringToClass(const NormalizedInstruction& i);
237 void emitStringToKnownClass(const NormalizedInstruction& i,
238 const StringData* clssName);
239 void emitObjToClass(const NormalizedInstruction& i);
240 void emitClsAndPals(const NormalizedInstruction& i);
242 template<int Arity> TCA emitNAryStub(Asm& a, CppCall c);
243 TCA emitUnaryStub(Asm& a, CppCall c);
244 TCA genericRefCountStub(Asm& a);
245 TCA genericRefCountStubRegs(Asm& a);
246 void emitFreeLocalsHelpers();
247 void emitGenericDecRefHelpers();
248 TCA emitPrologueRedispatch(Asm &a);
249 TCA emitFuncGuard(Asm& a, const Func *f);
250 template <bool reentrant>
251 void emitDerefStoreToLoc(PhysReg srcReg, const Location& destLoc);
253 void getInputsIntoXMMRegs(const NormalizedInstruction& ni,
254 PhysReg lr, PhysReg rr,
255 RegXMM lxmm, RegXMM rxmm);
256 void fpEq(const NormalizedInstruction& i, PhysReg lr, PhysReg rr);
257 void emitRB(Asm& a, Trace::RingBufferType t, SrcKey sk,
258 RegSet toSave = RegSet());
259 void emitRB(Asm& a, Trace::RingBufferType t, const char* msgm,
260 RegSet toSave = RegSet());
261 void newTuple(const NormalizedInstruction& i, unsigned n);
263 enum {
264 ArgDontAllocate = -1,
265 ArgAnyReg = -2
268 private:
269 template<typename L>
270 void translatorAssert(X64Assembler& a, ConditionCode cc,
271 const char* msg, L setup);
273 static uint64_t toStringHelper(ObjectData *obj);
274 void invalidateSrcKey(SrcKey sk);
275 bool dontGuardAnyInputs(Op op);
276 public:
277 template<typename T>
278 void invalidateSrcKeys(const T& keys) {
279 BlockingLeaseHolder writer(s_writeLease);
280 assert(writer);
281 for (typename T::const_iterator i = keys.begin(); i != keys.end(); ++i) {
282 invalidateSrcKey(*i);
286 void registerCatchTrace(CTCA ip, TCA trace);
287 TCA getCatchTrace(CTCA ip) const;
289 static void SEGVHandler(int signum, siginfo_t *info, void *ctx);
291 void fixupWork(VMExecutionContext* ec, ActRec* startRbp) const;
292 void fixup(VMExecutionContext* ec) const;
293 TCA getTranslatedCaller() const;
295 TCA getTopTranslation(SrcKey sk) {
296 return getSrcRec(sk)->getTopTranslation();
299 TCA getCallToExit() {
300 return m_callToExit;
303 TCA getRetFromInterpretedFrame() {
304 return m_retHelper;
307 TCA getRetFromInlinedFrame() {
308 return m_retInlHelper;
311 TCA getRetFromInterpretedGeneratorFrame() {
312 return m_genRetHelper;
315 inline bool isValidCodeAddress(TCA tca) const {
316 return tca >= ahot.code.base && tca < astubs.code.base + astubs.code.size;
319 // If we were to shove every little helper function into this class
320 // header, we'd spend the rest of our lives compiling. So, these public
321 // functions are for static helpers private to translator-x64.cpp. Be
322 // professional.
324 Asm& getAsm() { return a; }
325 void emitChainTo(SrcKey dest, bool isCall = false);
327 static bool isPseudoEvent(const char* event);
328 void getPerfCounters(Array& ret);
330 private:
331 virtual void syncWork();
333 public:
334 bool acquireWriteLease(bool blocking) {
335 return s_writeLease.acquire(blocking);
337 void dropWriteLease() {
338 s_writeLease.drop();
341 void emitGuardChecks(Asm& a, SrcKey, const ChangeMap&,
342 const RefDeps&, SrcRec&);
343 void irEmitResolvedDeps(const ChangeMap& resolvedDeps);
345 Debug::DebugInfo* getDebugInfo() { return &m_debugInfo; }
347 template<typename T, typename... Args>
348 T* allocData(Args&&... args) {
349 return m_globalData.alloc<T>(std::forward<Args>(args)...);
352 FreeStubList m_freeStubs;
353 bool freeRequestStub(TCA stub);
354 TCA getFreeStub();
355 bool checkTranslationLimit(SrcKey, const SrcRec&) const;
356 TranslateResult irTranslateTracelet(Tracelet& t);
358 void irAssertType(const Location& l, const RuntimeType& rtt);
359 void checkType(Asm&, const Location& l, const RuntimeType& rtt,
360 SrcRec& fail);
361 void irCheckType(Asm&, const Location& l, const RuntimeType& rtt,
362 SrcRec& fail);
364 void checkRefs(Asm&, SrcKey, const RefDeps&, SrcRec&);
366 void emitInlineReturn(Location retvalSrcLoc, int retvalSrcDisp);
367 void emitGenericReturn(bool noThis, int retvalSrcDisp);
368 void dumpStack(const char* msg, int offset) const;
370 void emitFallbackJmp(SrcRec& dest, ConditionCode cc = CC_NZ);
371 void emitFallbackJmp(Asm& as, SrcRec& dest, ConditionCode cc = CC_NZ);
372 void emitFallbackUncondJmp(Asm& as, SrcRec& dest);
373 void emitFallbackCondJmp(Asm& as, SrcRec& dest, ConditionCode cc);
374 void emitDebugPrint(Asm&, const char*,
375 PhysReg = reg::r13,
376 PhysReg = reg::r14,
377 PhysReg = reg::rax);
379 TCA emitServiceReq(ServiceRequest, int numArgs, ...);
380 TCA emitServiceReq(SRFlags flags, ServiceRequest, int numArgs, ...);
381 TCA emitServiceReqVA(SRFlags flags, ServiceRequest, int numArgs,
382 va_list args);
384 TCA emitRetFromInterpretedFrame();
385 TCA emitRetFromInterpretedGeneratorFrame();
386 void emitPopRetIntoActRec(Asm& a);
387 int32_t emitBindCall(SrcKey srcKey, const Func* funcd, int numArgs);
388 void emitCondJmp(SrcKey skTrue, SrcKey skFalse, ConditionCode cc);
390 TCA funcPrologue(Func* func, int nArgs, ActRec* ar = nullptr);
391 bool checkCachedPrologue(const Func* func, int param, TCA& plgOut) const;
392 SrcKey emitPrologue(Func* func, int nArgs);
393 static bool eagerRecord(const Func* func);
394 int32_t emitNativeImpl(const Func*, bool emitSavedRIPReturn);
395 void emitBindJ(Asm& a, ConditionCode cc, SrcKey dest,
396 ServiceRequest req);
397 void emitBindJmp(Asm& a, SrcKey dest,
398 ServiceRequest req = REQ_BIND_JMP);
399 void emitBindJcc(Asm& a, ConditionCode cc, SrcKey dest,
400 ServiceRequest req = REQ_BIND_JCC);
401 void emitBindJmp(SrcKey dest);
402 void emitBindCallHelper(SrcKey srcKey,
403 const Func* funcd,
404 int numArgs);
405 void emitIncCounter(TCA start, int cntOfs);
407 private:
408 void moveToAlign(Asm &aa, const size_t alignment = kJmpTargetAlign,
409 const bool unreachable = true);
410 static void smash(Asm &a, TCA src, TCA dest, bool isCall);
411 static void smashJmp(Asm &a, TCA src, TCA dest) {
412 smash(a, src, dest, false);
414 static void smashCall(Asm &a, TCA src, TCA dest) {
415 smash(a, src, dest, true);
418 TCA getTranslation(const TranslArgs& args);
419 TCA createTranslation(const TranslArgs& args);
420 TCA retranslate(const TranslArgs& args);
421 TCA translate(const TranslArgs& args);
422 void translateWork(const TranslArgs& args);
424 TCA lookupTranslation(SrcKey sk) const;
425 TCA retranslateOpt(TransID transId, bool align);
426 TCA retranslateAndPatchNoIR(SrcKey sk,
427 bool align,
428 TCA toSmash);
429 TCA bindJmp(TCA toSmash, SrcKey dest, ServiceRequest req, bool& smashed);
430 TCA bindJmpccFirst(TCA toSmash,
431 Offset offTrue, Offset offFalse,
432 bool toTake,
433 ConditionCode cc,
434 bool& smashed);
435 TCA bindJmpccSecond(TCA toSmash, const Offset off,
436 ConditionCode cc,
437 bool& smashed);
438 bool handleServiceRequest(TReqInfo&, TCA& start, SrcKey& sk);
440 void recordGdbTranslation(SrcKey sk, const Func* f,
441 const Asm& a,
442 const TCA start,
443 bool exit, bool inPrologue);
444 void recordGdbStub(const Asm& a, TCA start, const char* name);
445 void recordBCInstr(uint32_t op, const Asm& a, const TCA addr);
447 void emitStackCheck(int funcDepth, Offset pc);
448 void emitStackCheckDynamic(int numArgs, Offset pc);
449 void emitTestSurpriseFlags(Asm& a);
450 void emitCheckSurpriseFlagsEnter(bool inTracelet, Fixup fixup);
451 TCA emitTransCounterInc(Asm& a);
453 static void trimExtraArgs(ActRec* ar);
454 static int shuffleArgsForMagicCall(ActRec* ar);
455 static void setArgInActRec(ActRec* ar, int argNum, uint64_t datum,
456 DataType t);
458 static void fCallArrayHelper(const Offset pcOff, const Offset pcNext);
460 TCA getNativeTrampoline(TCA helperAddress);
461 TCA emitNativeTrampoline(TCA helperAddress);
463 public:
465 * enterTC is the main entry point for the translator from the
466 * bytecode interpreter (see enterVMWork). It operates on behalf of
467 * a given nested invocation of the intepreter (calling back into it
468 * as necessary for blocks that need to be interpreted).
470 * If start is not null, data will be used to initialize rStashedAr,
471 * to enable us to run a jitted prolog;
472 * otherwise, data should be a pointer to the SrcKey to start
473 * translating from.
475 * But don't call this directly, use one of the helpers below
477 void enterTC(TCA start, void* data);
478 void enterTCAtSrcKey(SrcKey& sk) {
479 enterTC(nullptr, &sk);
481 void enterTCAtProlog(ActRec *ar, TCA start) {
482 enterTC(start, ar);
484 void enterTCAfterProlog(TCA start) {
485 enterTC(start, nullptr);
488 TranslatorX64();
489 virtual ~TranslatorX64();
491 void initGdb();
492 static TranslatorX64* Get();
494 // Called before entering a new PHP "world."
495 void requestInit();
497 // Called at the end of eval()
498 void requestExit();
500 // Returns a string with cache usage information
501 virtual std::string getUsage();
502 virtual size_t getCodeSize();
503 virtual size_t getStubSize();
504 virtual size_t getTargetCacheSize();
506 // true iff calling thread is sole writer.
507 static bool canWrite() {
508 // We can get called early in boot, so allow null tx64.
509 return !tx64 || s_writeLease.amOwner();
512 // Returns true on success
513 bool dumpTC(bool ignoreLease = false);
515 // Returns true on success
516 bool dumpTCCode(const char* filename);
518 // Returns true on success
519 bool dumpTCData();
521 // Debugging interfaces to prevent tampering with code.
522 void protectCode();
523 void unprotectCode();
525 int numTranslations(SrcKey sk) const;
526 private:
527 virtual bool addDbgGuards(const Unit* unit);
528 virtual bool addDbgGuard(const Func* func, Offset offset);
529 void addDbgGuardImpl(SrcKey sk, SrcRec& sr);
531 public: // Only for HackIR
532 void emitReqRetransNoIR(Asm& as, const SrcKey& sk);
534 private:
535 // asize + astubssize + gdatasize + trampolinesblocksize
536 size_t m_totalSize;
541 * RAII bookmark for temporarily rewinding a.code.frontier.
543 class CodeCursor {
544 typedef X64Assembler Asm;
545 Asm& m_a;
546 TCA m_oldFrontier;
547 public:
548 CodeCursor(Asm& a, TCA newFrontier) :
549 m_a(a), m_oldFrontier(a.code.frontier) {
550 assert(TranslatorX64::canWrite());
551 m_a.code.frontier = newFrontier;
552 TRACE_MOD(Trace::trans, 1, "RewindTo: %p (from %p)\n",
553 m_a.code.frontier, m_oldFrontier);
555 ~CodeCursor() {
556 assert(TranslatorX64::canWrite());
557 m_a.code.frontier = m_oldFrontier;
558 TRACE_MOD(Trace::trans, 1, "Restore: %p\n",
559 m_a.code.frontier);
563 const size_t kTrampolinesBlockSize = 8 << 12;
565 // minimum length in bytes of each trampoline code sequence
566 // Note that if stats is on, then this size is ~24 bytes due to the
567 // instrumentation code that counts the number of calls through each
568 // trampoline
569 const size_t kMinPerTrampolineSize = 11;
571 const size_t kMaxNumTrampolines = kTrampolinesBlockSize /
572 kMinPerTrampolineSize;
574 void fcallHelperThunk() asm ("__fcallHelperThunk");
575 void funcBodyHelperThunk() asm ("__funcBodyHelperThunk");
576 void functionEnterHelper(const ActRec* ar);
577 int64_t decodeCufIterHelper(Iter* it, TypedValue func);
579 // These could be static but are used in hopt/codegen.cpp
580 void raiseUndefVariable(StringData* nm);
581 void defFuncHelper(Func *f);
582 Instance* newInstanceHelper(Class* cls, int numArgs, ActRec* ar,
583 ActRec* prevAr);
584 Instance* newInstanceHelperCached(Class** classCache,
585 const StringData* clsName, int numArgs,
586 ActRec* ar, ActRec* prevAr);
587 Instance* newInstanceHelperNoCtorCached(Class** classCache,
588 const StringData* clsName);
590 bool isNormalPropertyAccess(const NormalizedInstruction& i,
591 int propInput,
592 int objInput);
594 bool mInstrHasUnknownOffsets(const NormalizedInstruction& i,
595 Class* contextClass);
597 struct PropInfo {
598 PropInfo()
599 : offset(-1)
600 , hphpcType(KindOfInvalid)
602 explicit PropInfo(int offset, DataType hphpcType)
603 : offset(offset)
604 , hphpcType(hphpcType)
607 int offset;
608 DataType hphpcType;
611 PropInfo getPropertyOffset(const NormalizedInstruction& ni,
612 Class* contextClass,
613 const Class*& baseClass,
614 const MInstrInfo& mii,
615 unsigned mInd, unsigned iInd);
616 PropInfo getFinalPropertyOffset(const NormalizedInstruction&,
617 Class* contextClass,
618 const MInstrInfo&);
620 bool isSupportedCGetM(const NormalizedInstruction& i);
621 TXFlags planInstrAdd_Int(const NormalizedInstruction& i);
622 TXFlags planInstrAdd_Array(const NormalizedInstruction& i);
623 void dumpTranslationInfo(const Tracelet& t, TCA postGuards);
625 // SpaceRecorder is used in translator-x64.cpp and in hopt/irtranslator.cpp
626 // RAII logger for TC space consumption.
627 struct SpaceRecorder {
628 const char *m_name;
629 const X64Assembler m_a;
630 // const X64Assembler& m_a;
631 const uint8_t *m_start;
632 SpaceRecorder(const char* name, const X64Assembler& a) :
633 m_name(name), m_a(a), m_start(a.code.frontier)
635 ~SpaceRecorder() {
636 if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
637 ptrdiff_t diff = m_a.code.frontier - m_start;
638 if (diff) {
639 Trace::traceRelease("TCSpace %10s %3" PRId64 "\n", m_name, diff);
645 typedef const int COff; // Const offsets
649 #endif