2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
16 #ifndef incl_HPHP_RUNTIME_VM_TRANSLATOR_X64_H_
17 #define incl_HPHP_RUNTIME_VM_TRANSLATOR_X64_H_
21 #include <boost/noncopyable.hpp>
23 #include "hphp/runtime/vm/bytecode.h"
24 #include "hphp/runtime/vm/jit/translator.h"
25 #include "hphp/util/asm-x64.h"
26 #include "hphp/runtime/vm/jit/unwind-x64.h"
27 #include "tbb/concurrent_hash_map.h"
28 #include "hphp/util/ringbuffer.h"
29 #include "hphp/runtime/vm/debug/debug.h"
30 #include "hphp/runtime/vm/jit/abi-x64.h"
32 namespace HPHP
{ class ExecutionContext
; }
34 namespace HPHP
{ namespace JIT
{
42 namespace HPHP
{ namespace Transl
{
44 struct TraceletCounters
{
45 uint64_t m_numEntered
, m_numExecuted
;
48 struct TraceletCountersVec
{
50 TraceletCounters
*m_elms
;
53 TraceletCountersVec() : m_size(0), m_elms(nullptr), m_lock() { }
61 static const uint64_t kStubFree
= 0;
63 FreeStubList() : m_list(nullptr) {}
69 explicit CppCall(void *p
) : m_kind(Direct
), m_fptr(p
) {}
70 explicit CppCall(int off
) : m_kind(Virtual
), m_offset(off
) {}
71 CppCall(CppCall
const&) = default;
73 bool isDirect() const { return m_kind
== Direct
; }
74 bool isVirtual() const { return m_kind
== Virtual
; }
76 const void* getAddress() const { return m_fptr
; }
77 int getOffset() const { return m_offset
; }
80 enum { Direct
, Virtual
} m_kind
;
88 extern __thread TranslatorX64
* tx64
;
90 extern void* interpOneEntryPoints
[];
92 extern "C" TCA
funcBodyHelper(ActRec
* fp
);
97 static const int kNumFreeLocalsHelpers
= 9;
99 typedef X64Assembler Asm
;
101 constexpr size_t kJmpTargetAlign
= 16;
102 constexpr size_t kNonFallthroughAlign
= 64;
103 constexpr int kJmpLen
= 5;
104 constexpr int kCallLen
= 5;
105 constexpr int kJmpccLen
= 6;
106 constexpr int kJmpImmBytes
= 4;
107 constexpr int kJcc8Len
= 3;
108 constexpr int kLeaRipLen
= 7;
109 constexpr int kTestRegRegLen
= 3;
110 constexpr int kTestImmRegLen
= 5; // only for rax -- special encoding
111 // Cache alignment is required for mutable instructions to make sure
112 // mutations don't "tear" on remote cpus.
113 constexpr size_t kX64CacheLineSize
= 64;
114 constexpr size_t kX64CacheLineMask
= kX64CacheLineSize
- 1;
116 enum class TestAndSmashFlags
{
121 void prepareForTestAndSmash(Asm
&, int testBytes
, TestAndSmashFlags flags
);
122 void prepareForSmash(Asm
&, int nBytes
, int offset
= 0);
123 bool isSmashable(Address frontier
, int nBytes
, int offset
= 0);
125 class TranslatorX64
: public Translator
126 , boost::noncopyable
{
127 friend class SrcRec
; // so it can smash code.
128 friend class SrcDB
; // For write lock and code invalidation.
129 friend class Tx64Reaper
;
130 friend class HPHP::JIT::CodeGenerator
;
132 typedef tbb::concurrent_hash_map
<TCA
, TCA
> SignalStubMap
;
133 typedef void (*sigaction_t
)(int, siginfo_t
*, void*);
135 typedef X64Assembler Asm
;
139 AHotSelector(TranslatorX64
* tx
, bool hot
) :
140 m_tx(tx
), m_hot(hot
&&
141 tx
->ahot
.code
.base
+ tx
->ahot
.code
.size
-
142 tx
->ahot
.code
.frontier
> 8192 &&
143 tx
->a
.code
.base
!= tx
->ahot
.code
.base
) {
151 m_tx
->ahot
= m_tx
->a
;
165 PointerMap trampolineMap
;
166 int m_numNativeTrampolines
;
167 size_t m_trampolineSize
; // size of each trampoline
169 SignalStubMap m_segvStubs
;
170 sigaction_t m_segvChain
;
175 TCA m_stackOverflowHelper
;
177 TCA m_dtorGenericStub
;
178 TCA m_dtorGenericStubRegs
;
179 TCA m_dtorStubs
[kDestrTableSize
];
181 TCA m_funcPrologueRedispatch
;
183 TCA m_freeManyLocalsHelper
;
184 TCA m_freeLocalsHelpers
[kNumFreeLocalsHelpers
];
186 DataBlock m_globalData
;
188 // Data structures for HHIR-based translation
189 uint64_t m_numHHIRTrans
;
191 virtual void traceCodeGen();
194 UnwindInfoHandle m_unwindRegistrar
;
195 CatchTraceMap m_catchTraceMap
;
196 std::vector
<TransBCMapping
> m_bcMap
;
198 Debug::DebugInfo m_debugInfo
;
201 int64_t m_createdTime
;
203 struct PendingFixup
{
207 PendingFixup(TCA tca
, Fixup fixup
) :
208 m_tca(tca
), m_fixup(fixup
) { }
210 vector
<PendingFixup
> m_pendingFixups
;
212 void drawCFG(std::ofstream
& out
) const;
213 static vector
<PhysReg
> x64TranslRegs();
215 Asm
& getAsmFor(TCA addr
) { return asmChoose(addr
, a
, ahot
, astubs
); }
216 void emitIncRef(X64Assembler
&a
, PhysReg base
, DataType dtype
);
217 void emitIncRef(PhysReg base
, DataType
);
218 void emitIncRefGenericRegSafe(PhysReg base
, int disp
, PhysReg tmp
);
219 static CppCall
getDtorCall(DataType type
);
220 void emitCopy(PhysReg srcCell
, int disp
, PhysReg destCell
);
222 void emitThisCheck(const NormalizedInstruction
& i
, PhysReg reg
);
225 void emitCall(Asm
& a
, TCA dest
);
226 void emitCall(Asm
& a
, CppCall call
);
227 TCA
getCallArrayProlog(Func
* func
);
230 void translateClassExistsImpl(const Tracelet
& t
,
231 const NormalizedInstruction
& i
,
233 void recordSyncPoint(Asm
& a
, Offset pcOff
, Offset spOff
);
234 void emitEagerSyncPoint(Asm
& a
, const Opcode
* pc
, const Offset spDiff
);
235 void recordIndirectFixup(CTCA addr
, int dwordsPushed
);
236 void emitStringToClass(const NormalizedInstruction
& i
);
237 void emitStringToKnownClass(const NormalizedInstruction
& i
,
238 const StringData
* clssName
);
239 void emitObjToClass(const NormalizedInstruction
& i
);
240 void emitClsAndPals(const NormalizedInstruction
& i
);
242 template<int Arity
> TCA
emitNAryStub(Asm
& a
, CppCall c
);
243 TCA
emitUnaryStub(Asm
& a
, CppCall c
);
244 TCA
genericRefCountStub(Asm
& a
);
245 TCA
genericRefCountStubRegs(Asm
& a
);
246 void emitFreeLocalsHelpers();
247 void emitGenericDecRefHelpers();
248 TCA
emitPrologueRedispatch(Asm
&a
);
249 TCA
emitFuncGuard(Asm
& a
, const Func
*f
);
250 template <bool reentrant
>
251 void emitDerefStoreToLoc(PhysReg srcReg
, const Location
& destLoc
);
253 void getInputsIntoXMMRegs(const NormalizedInstruction
& ni
,
254 PhysReg lr
, PhysReg rr
,
255 RegXMM lxmm
, RegXMM rxmm
);
256 void fpEq(const NormalizedInstruction
& i
, PhysReg lr
, PhysReg rr
);
257 void emitRB(Asm
& a
, Trace::RingBufferType t
, SrcKey sk
,
258 RegSet toSave
= RegSet());
259 void emitRB(Asm
& a
, Trace::RingBufferType t
, const char* msgm
,
260 RegSet toSave
= RegSet());
261 void newTuple(const NormalizedInstruction
& i
, unsigned n
);
264 ArgDontAllocate
= -1,
270 void translatorAssert(X64Assembler
& a
, ConditionCode cc
,
271 const char* msg
, L setup
);
273 static uint64_t toStringHelper(ObjectData
*obj
);
274 void invalidateSrcKey(SrcKey sk
);
275 bool dontGuardAnyInputs(Op op
);
278 void invalidateSrcKeys(const T
& keys
) {
279 BlockingLeaseHolder
writer(s_writeLease
);
281 for (typename
T::const_iterator i
= keys
.begin(); i
!= keys
.end(); ++i
) {
282 invalidateSrcKey(*i
);
286 void registerCatchTrace(CTCA ip
, TCA trace
);
287 TCA
getCatchTrace(CTCA ip
) const;
289 static void SEGVHandler(int signum
, siginfo_t
*info
, void *ctx
);
291 void fixupWork(VMExecutionContext
* ec
, ActRec
* startRbp
) const;
292 void fixup(VMExecutionContext
* ec
) const;
293 TCA
getTranslatedCaller() const;
295 TCA
getTopTranslation(SrcKey sk
) {
296 return getSrcRec(sk
)->getTopTranslation();
299 TCA
getCallToExit() {
303 TCA
getRetFromInterpretedFrame() {
307 TCA
getRetFromInlinedFrame() {
308 return m_retInlHelper
;
311 TCA
getRetFromInterpretedGeneratorFrame() {
312 return m_genRetHelper
;
315 inline bool isValidCodeAddress(TCA tca
) const {
316 return tca
>= ahot
.code
.base
&& tca
< astubs
.code
.base
+ astubs
.code
.size
;
319 // If we were to shove every little helper function into this class
320 // header, we'd spend the rest of our lives compiling. So, these public
321 // functions are for static helpers private to translator-x64.cpp. Be
324 Asm
& getAsm() { return a
; }
325 void emitChainTo(SrcKey dest
, bool isCall
= false);
327 static bool isPseudoEvent(const char* event
);
328 void getPerfCounters(Array
& ret
);
331 virtual void syncWork();
334 bool acquireWriteLease(bool blocking
) {
335 return s_writeLease
.acquire(blocking
);
337 void dropWriteLease() {
341 void emitGuardChecks(Asm
& a
, SrcKey
, const ChangeMap
&,
342 const RefDeps
&, SrcRec
&);
343 void irEmitResolvedDeps(const ChangeMap
& resolvedDeps
);
345 Debug::DebugInfo
* getDebugInfo() { return &m_debugInfo
; }
347 template<typename T
, typename
... Args
>
348 T
* allocData(Args
&&... args
) {
349 return m_globalData
.alloc
<T
>(std::forward
<Args
>(args
)...);
352 FreeStubList m_freeStubs
;
353 bool freeRequestStub(TCA stub
);
355 bool checkTranslationLimit(SrcKey
, const SrcRec
&) const;
356 TranslateResult
irTranslateTracelet(Tracelet
& t
);
358 void irAssertType(const Location
& l
, const RuntimeType
& rtt
);
359 void checkType(Asm
&, const Location
& l
, const RuntimeType
& rtt
,
361 void irCheckType(Asm
&, const Location
& l
, const RuntimeType
& rtt
,
364 void checkRefs(Asm
&, SrcKey
, const RefDeps
&, SrcRec
&);
366 void emitInlineReturn(Location retvalSrcLoc
, int retvalSrcDisp
);
367 void emitGenericReturn(bool noThis
, int retvalSrcDisp
);
368 void dumpStack(const char* msg
, int offset
) const;
370 void emitFallbackJmp(SrcRec
& dest
, ConditionCode cc
= CC_NZ
);
371 void emitFallbackJmp(Asm
& as
, SrcRec
& dest
, ConditionCode cc
= CC_NZ
);
372 void emitFallbackUncondJmp(Asm
& as
, SrcRec
& dest
);
373 void emitFallbackCondJmp(Asm
& as
, SrcRec
& dest
, ConditionCode cc
);
374 void emitDebugPrint(Asm
&, const char*,
379 TCA
emitServiceReq(ServiceRequest
, int numArgs
, ...);
380 TCA
emitServiceReq(SRFlags flags
, ServiceRequest
, int numArgs
, ...);
381 TCA
emitServiceReqVA(SRFlags flags
, ServiceRequest
, int numArgs
,
384 TCA
emitRetFromInterpretedFrame();
385 TCA
emitRetFromInterpretedGeneratorFrame();
386 void emitPopRetIntoActRec(Asm
& a
);
387 int32_t emitBindCall(SrcKey srcKey
, const Func
* funcd
, int numArgs
);
388 void emitCondJmp(SrcKey skTrue
, SrcKey skFalse
, ConditionCode cc
);
390 TCA
funcPrologue(Func
* func
, int nArgs
, ActRec
* ar
= nullptr);
391 bool checkCachedPrologue(const Func
* func
, int param
, TCA
& plgOut
) const;
392 SrcKey
emitPrologue(Func
* func
, int nArgs
);
393 static bool eagerRecord(const Func
* func
);
394 int32_t emitNativeImpl(const Func
*, bool emitSavedRIPReturn
);
395 void emitBindJ(Asm
& a
, ConditionCode cc
, SrcKey dest
,
397 void emitBindJmp(Asm
& a
, SrcKey dest
,
398 ServiceRequest req
= REQ_BIND_JMP
);
399 void emitBindJcc(Asm
& a
, ConditionCode cc
, SrcKey dest
,
400 ServiceRequest req
= REQ_BIND_JCC
);
401 void emitBindJmp(SrcKey dest
);
402 void emitBindCallHelper(SrcKey srcKey
,
405 void emitIncCounter(TCA start
, int cntOfs
);
408 void moveToAlign(Asm
&aa
, const size_t alignment
= kJmpTargetAlign
,
409 const bool unreachable
= true);
410 static void smash(Asm
&a
, TCA src
, TCA dest
, bool isCall
);
411 static void smashJmp(Asm
&a
, TCA src
, TCA dest
) {
412 smash(a
, src
, dest
, false);
414 static void smashCall(Asm
&a
, TCA src
, TCA dest
) {
415 smash(a
, src
, dest
, true);
418 TCA
getTranslation(const TranslArgs
& args
);
419 TCA
createTranslation(const TranslArgs
& args
);
420 TCA
retranslate(const TranslArgs
& args
);
421 TCA
translate(const TranslArgs
& args
);
422 void translateWork(const TranslArgs
& args
);
424 TCA
lookupTranslation(SrcKey sk
) const;
425 TCA
retranslateOpt(TransID transId
, bool align
);
426 TCA
retranslateAndPatchNoIR(SrcKey sk
,
429 TCA
bindJmp(TCA toSmash
, SrcKey dest
, ServiceRequest req
, bool& smashed
);
430 TCA
bindJmpccFirst(TCA toSmash
,
431 Offset offTrue
, Offset offFalse
,
435 TCA
bindJmpccSecond(TCA toSmash
, const Offset off
,
438 bool handleServiceRequest(TReqInfo
&, TCA
& start
, SrcKey
& sk
);
440 void recordGdbTranslation(SrcKey sk
, const Func
* f
,
443 bool exit
, bool inPrologue
);
444 void recordGdbStub(const Asm
& a
, TCA start
, const char* name
);
445 void recordBCInstr(uint32_t op
, const Asm
& a
, const TCA addr
);
447 void emitStackCheck(int funcDepth
, Offset pc
);
448 void emitStackCheckDynamic(int numArgs
, Offset pc
);
449 void emitTestSurpriseFlags(Asm
& a
);
450 void emitCheckSurpriseFlagsEnter(bool inTracelet
, Fixup fixup
);
451 TCA
emitTransCounterInc(Asm
& a
);
453 static void trimExtraArgs(ActRec
* ar
);
454 static int shuffleArgsForMagicCall(ActRec
* ar
);
455 static void setArgInActRec(ActRec
* ar
, int argNum
, uint64_t datum
,
458 static void fCallArrayHelper(const Offset pcOff
, const Offset pcNext
);
460 TCA
getNativeTrampoline(TCA helperAddress
);
461 TCA
emitNativeTrampoline(TCA helperAddress
);
465 * enterTC is the main entry point for the translator from the
466 * bytecode interpreter (see enterVMWork). It operates on behalf of
467 * a given nested invocation of the intepreter (calling back into it
468 * as necessary for blocks that need to be interpreted).
470 * If start is not null, data will be used to initialize rStashedAr,
471 * to enable us to run a jitted prolog;
472 * otherwise, data should be a pointer to the SrcKey to start
475 * But don't call this directly, use one of the helpers below
477 void enterTC(TCA start
, void* data
);
478 void enterTCAtSrcKey(SrcKey
& sk
) {
479 enterTC(nullptr, &sk
);
481 void enterTCAtProlog(ActRec
*ar
, TCA start
) {
484 void enterTCAfterProlog(TCA start
) {
485 enterTC(start
, nullptr);
489 virtual ~TranslatorX64();
492 static TranslatorX64
* Get();
494 // Called before entering a new PHP "world."
497 // Called at the end of eval()
500 // Returns a string with cache usage information
501 virtual std::string
getUsage();
502 virtual size_t getCodeSize();
503 virtual size_t getStubSize();
504 virtual size_t getTargetCacheSize();
506 // true iff calling thread is sole writer.
507 static bool canWrite() {
508 // We can get called early in boot, so allow null tx64.
509 return !tx64
|| s_writeLease
.amOwner();
512 // Returns true on success
513 bool dumpTC(bool ignoreLease
= false);
515 // Returns true on success
516 bool dumpTCCode(const char* filename
);
518 // Returns true on success
521 // Debugging interfaces to prevent tampering with code.
523 void unprotectCode();
525 int numTranslations(SrcKey sk
) const;
527 virtual bool addDbgGuards(const Unit
* unit
);
528 virtual bool addDbgGuard(const Func
* func
, Offset offset
);
529 void addDbgGuardImpl(SrcKey sk
, SrcRec
& sr
);
531 public: // Only for HackIR
532 void emitReqRetransNoIR(Asm
& as
, const SrcKey
& sk
);
535 // asize + astubssize + gdatasize + trampolinesblocksize
541 * RAII bookmark for temporarily rewinding a.code.frontier.
544 typedef X64Assembler Asm
;
548 CodeCursor(Asm
& a
, TCA newFrontier
) :
549 m_a(a
), m_oldFrontier(a
.code
.frontier
) {
550 assert(TranslatorX64::canWrite());
551 m_a
.code
.frontier
= newFrontier
;
552 TRACE_MOD(Trace::trans
, 1, "RewindTo: %p (from %p)\n",
553 m_a
.code
.frontier
, m_oldFrontier
);
556 assert(TranslatorX64::canWrite());
557 m_a
.code
.frontier
= m_oldFrontier
;
558 TRACE_MOD(Trace::trans
, 1, "Restore: %p\n",
563 const size_t kTrampolinesBlockSize
= 8 << 12;
565 // minimum length in bytes of each trampoline code sequence
566 // Note that if stats is on, then this size is ~24 bytes due to the
567 // instrumentation code that counts the number of calls through each
569 const size_t kMinPerTrampolineSize
= 11;
571 const size_t kMaxNumTrampolines
= kTrampolinesBlockSize
/
572 kMinPerTrampolineSize
;
574 void fcallHelperThunk() asm ("__fcallHelperThunk");
575 void funcBodyHelperThunk() asm ("__funcBodyHelperThunk");
576 void functionEnterHelper(const ActRec
* ar
);
577 int64_t decodeCufIterHelper(Iter
* it
, TypedValue func
);
579 // These could be static but are used in hopt/codegen.cpp
580 void raiseUndefVariable(StringData
* nm
);
581 void defFuncHelper(Func
*f
);
582 Instance
* newInstanceHelper(Class
* cls
, int numArgs
, ActRec
* ar
,
584 Instance
* newInstanceHelperCached(Class
** classCache
,
585 const StringData
* clsName
, int numArgs
,
586 ActRec
* ar
, ActRec
* prevAr
);
587 Instance
* newInstanceHelperNoCtorCached(Class
** classCache
,
588 const StringData
* clsName
);
590 bool isNormalPropertyAccess(const NormalizedInstruction
& i
,
594 bool mInstrHasUnknownOffsets(const NormalizedInstruction
& i
,
595 Class
* contextClass
);
600 , hphpcType(KindOfInvalid
)
602 explicit PropInfo(int offset
, DataType hphpcType
)
604 , hphpcType(hphpcType
)
611 PropInfo
getPropertyOffset(const NormalizedInstruction
& ni
,
613 const Class
*& baseClass
,
614 const MInstrInfo
& mii
,
615 unsigned mInd
, unsigned iInd
);
616 PropInfo
getFinalPropertyOffset(const NormalizedInstruction
&,
620 bool isSupportedCGetM(const NormalizedInstruction
& i
);
621 TXFlags
planInstrAdd_Int(const NormalizedInstruction
& i
);
622 TXFlags
planInstrAdd_Array(const NormalizedInstruction
& i
);
623 void dumpTranslationInfo(const Tracelet
& t
, TCA postGuards
);
625 // SpaceRecorder is used in translator-x64.cpp and in hopt/irtranslator.cpp
626 // RAII logger for TC space consumption.
627 struct SpaceRecorder
{
629 const X64Assembler m_a
;
630 // const X64Assembler& m_a;
631 const uint8_t *m_start
;
632 SpaceRecorder(const char* name
, const X64Assembler
& a
) :
633 m_name(name
), m_a(a
), m_start(a
.code
.frontier
)
636 if (Trace::moduleEnabledRelease(Trace::tcspace
, 1)) {
637 ptrdiff_t diff
= m_a
.code
.frontier
- m_start
;
639 Trace::traceRelease("TCSpace %10s %3" PRId64
"\n", m_name
, diff
);
645 typedef const int COff
; // Const offsets