2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
17 #include "hphp/runtime/vm/jit/target-cache.h"
19 #include "hphp/runtime/base/builtin-functions.h"
20 #include "hphp/runtime/base/execution-context.h"
21 #include "hphp/runtime/base/runtime-error.h"
22 #include "hphp/runtime/base/runtime-option.h"
23 #include "hphp/runtime/base/stats.h"
24 #include "hphp/runtime/base/strings.h"
26 #include "hphp/runtime/vm/jit/smashable-instr.h"
27 #include "hphp/runtime/vm/jit/tc-internal.h"
28 #include "hphp/runtime/vm/jit/tc.h"
29 #include "hphp/runtime/vm/jit/translator-inline.h"
30 #include "hphp/runtime/vm/jit/translator-runtime.h"
31 #include "hphp/runtime/vm/jit/write-lease.h"
33 #include "hphp/runtime/vm/method-lookup.h"
34 #include "hphp/runtime/vm/treadmill.h"
35 #include "hphp/runtime/vm/unit-util.h"
37 #include "hphp/util/text-util.h"
45 namespace HPHP
{ namespace jit
{
47 TRACE_SET_MOD(targetcache
);
49 //////////////////////////////////////////////////////////////////////
53 const StaticString
s_call("__call");
55 inline bool stringMatches(const StringData
* rowString
, const StringData
* sd
) {
58 rowString
->data() == sd
->data() ||
59 (rowString
->hash() == sd
->hash() &&
60 rowString
->same(sd
)));
63 template<class T
= void>
64 T
* handleToPtr(rds::Handle h
) {
65 return (T
*)((char*)rds::tl_base
+ h
);
69 typename
Cache::Pair
* keyToPair(Cache
* cache
, const StringData
* k
) {
70 assertx(folly::isPowTwo(Cache::kNumLines
));
71 return cache
->m_pairs
+ (k
->hash() & (Cache::kNumLines
- 1));
76 //////////////////////////////////////////////////////////////////////
79 // Set of FuncCache handles for dynamic function callsites, used for
80 // invalidation when a function is renamed.
81 static std::mutex funcCacheMutex
;
82 static std::vector
<rds::Link
<FuncCache
, true /* normal_only */>>
85 rds::Handle
FuncCache::alloc() {
86 auto const link
= rds::alloc
<FuncCache
,sizeof(Pair
),true>();
87 std::lock_guard
<std::mutex
> g(funcCacheMutex
);
88 funcCacheEntries
.push_back(link
);
92 void FuncCache::lookup(rds::Handle handle
,
96 auto const thiz
= handleToPtr
<FuncCache
>(handle
);
97 if (!rds::isHandleInit(handle
, rds::NormalTag
{})) {
98 for (std::size_t i
= 0; i
< FuncCache::kNumLines
; ++i
) {
99 thiz
->m_pairs
[i
].m_key
= nullptr;
100 thiz
->m_pairs
[i
].m_value
= nullptr;
102 rds::initHandle(handle
);
104 auto const pair
= keyToPair(thiz
, sd
);
105 const StringData
* pairSd
= pair
->m_key
;
106 if (!stringMatches(pairSd
, sd
)) {
107 // Miss. Does it actually exist?
108 auto const* func
= Unit::lookupDynCallFunc(sd
);
109 if (UNLIKELY(!func
)) {
110 ObjectData
*this_
= nullptr;
111 Class
* self_
= nullptr;
112 StringData
* inv
= nullptr;
114 func
= vm_decode_function(
121 DecodeFlags::NoWarn
);
123 raise_call_to_undefined(sd
);
126 *arPreliveOverwriteCells(ar
) = make_tv
<KindOfString
>(sd
);
133 this_
->incRefCount();
134 if (UNLIKELY(inv
!= nullptr)) ar
->setMagicDispatch(inv
);
140 if (UNLIKELY(inv
!= nullptr)) ar
->setMagicDispatch(inv
);
144 assertx(!func
->implCls());
147 const_cast<StringData
*>(func
->displayName()); // use a static name
148 pair
->m_value
= func
;
150 ar
->m_func
= pair
->m_value
;
152 assertx(stringMatches(pair
->m_key
, pair
->m_value
->displayName()));
153 pair
->m_value
->validate();
156 void invalidateForRenameFunction(const StringData
* name
) {
158 std::lock_guard
<std::mutex
> g(funcCacheMutex
);
159 for (auto& h
: funcCacheEntries
) {
160 if (h
.isInit()) h
.markUninit();
164 //////////////////////////////////////////////////////////////////////
167 rds::Handle
ClassCache::alloc() {
168 return rds::alloc
<ClassCache
,sizeof(Pair
)>().handle();
171 const Class
* ClassCache::lookup(rds::Handle handle
, StringData
* name
) {
172 auto const thiz
= handleToPtr
<ClassCache
>(handle
);
173 if (!rds::isHandleInit(handle
, rds::NormalTag
{})) {
174 for (std::size_t i
= 0; i
< ClassCache::kNumLines
; ++i
) {
175 thiz
->m_pairs
[i
].m_key
= nullptr;
176 thiz
->m_pairs
[i
].m_value
= nullptr;
178 rds::initHandle(handle
);
180 auto const pair
= keyToPair(thiz
, name
);
181 const StringData
* pairSd
= pair
->m_key
;
182 if (!stringMatches(pairSd
, name
)) {
183 TRACE(1, "ClassCache miss: %s\n", name
->data());
184 Class
* c
= Unit::loadClass(name
);
186 raise_error(Strings::UNKNOWN_CLASS
, name
->data());
188 if (pair
->m_key
) decRefStr(pair
->m_key
);
193 TRACE(1, "ClassCache hit: %s\n", name
->data());
195 return pair
->m_value
;
198 //=============================================================================
201 namespace MethodCache
{
204 ///////////////////////////////////////////////////////////////////////////////
206 [[noreturn
]] NEVER_INLINE
207 void raiseFatal(ActRec
* ar
, Class
* cls
, StringData
* name
, Class
* ctx
) {
209 lookupMethodCtx(cls
, name
, ctx
, CallType::ObjMethod
, true /* raise */);
212 // The jit stored an ObjectData in the ActRec, but we didn't set
214 auto const obj
= ar
->getThisUnsafe();
215 *arPreliveOverwriteCells(ar
) = make_tv
<KindOfObject
>(obj
);
221 void nullFunc(ActRec
* ar
, StringData
* name
) {
223 raise_warning("Invalid argument: function: method '%s' not found",
225 ar
->m_func
= SystemLib::s_nullFunc
;
226 auto const obj
= ar
->getThisUnsafe();
230 // The jit stored an ObjectData in the ActRec, but we didn't set
232 auto const obj
= ar
->getThisUnsafe();
233 *arPreliveOverwriteCells(ar
) = make_tv
<KindOfObject
>(obj
);
240 void lookup(Entry
* mce
, ActRec
* ar
, StringData
* name
, Class
* cls
, Class
* ctx
) {
241 auto func
= lookupMethodCtx(
249 if (UNLIKELY(!func
)) {
250 func
= cls
->lookupMethod(s_call
.get());
251 if (UNLIKELY(!func
)) {
252 if (fatal
) return raiseFatal(ar
, cls
, name
, ctx
);
253 return nullFunc(ar
, name
);
255 ar
->setMagicDispatch(name
);
256 assert(!(func
->attrs() & AttrStatic
));
258 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
) | 0x1u
;
263 auto const isStatic
= func
->isStaticInPrologue();
264 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
) | uintptr_t{isStatic
} << 1;
268 if (UNLIKELY(isStatic
)) {
269 auto const obj
= ar
->getThis();
277 void readMagicOrStatic(Entry
* mce
,
283 auto const storedClass
= reinterpret_cast<Class
*>(mceKey
& ~0x3u
);
284 if (storedClass
!= cls
) {
285 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
288 auto const mceValue
= mce
->m_value
;
289 ar
->m_func
= mceValue
;
291 auto const isMagic
= mceKey
& 0x1u
;
292 if (UNLIKELY(isMagic
)) {
293 ar
->setMagicDispatch(name
);
294 assertx(!(mceKey
& 0x2u
));
298 assertx(mceKey
& 0x2u
);
299 auto const obj
= ar
->getThis();
304 template <bool fatal
>
306 readPublicStatic(Entry
* mce
, ActRec
* ar
, Class
* cls
, const Func
* /*cand*/) {
307 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
) | 0x2u
;
308 auto const obj
= ar
->getThis();
313 ///////////////////////////////////////////////////////////////////////////////
317 void handleSlowPath(rds::Handle mce_handle
,
322 uintptr_t mcePrime
) {
323 assertx(ActRec::checkThis(ar
->getThisUnsafe()));
324 assertx(ar
->getThisUnsafe()->getVMClass() == cls
);
325 assertx(name
->isStatic());
327 auto const mce
= &rds::handleToRef
<Entry
>(mce_handle
);
328 if (!rds::isHandleInit(mce_handle
, rds::NormalTag
{})) {
330 mce
->m_value
= nullptr;
331 rds::initHandle(mce_handle
);
333 assertx(IMPLIES(mce
->m_key
, mce
->m_value
));
335 // Check for a hit in the request local cache---since we've failed
336 // on the immediate smashed in the TC.
337 auto const mceKey
= mce
->m_key
;
338 if (LIKELY(mceKey
== reinterpret_cast<uintptr_t>(cls
))) {
339 ar
->m_func
= mce
->m_value
;
343 // If the request local cache isn't filled, try to use the Func*
344 // from the TC's mcePrime as a starting point.
345 const Func
* mceValue
;
346 if (UNLIKELY(!mceKey
)) {
347 // If the low bit is set in mcePrime, we're in the middle of
348 // smashing immediates into the TC from the handlePrimeCacheInit,
349 // and the upper bits is not yet a valid Func*.
351 // We're assuming that writes to executable code may be seen out
352 // of order (i.e. it may call this function with the old
353 // immediate), so we check this bit to ensure we don't try to
354 // treat the immediate as a real Func* if it isn't yet.
355 if (mcePrime
& 0x1) {
356 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
358 mceValue
= reinterpret_cast<const Func
*>(mcePrime
>> 32);
359 if (UNLIKELY(!mceValue
)) {
360 // The inline Func* might be null if it was uncacheable (not
362 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
364 mce
->m_value
= mceValue
; // below assumes this is already in local cache
366 if (UNLIKELY(mceKey
& 0x3)) {
367 return readMagicOrStatic
<fatal
>(mce
, ar
, name
, cls
, ctx
, mceKey
);
369 mceValue
= mce
->m_value
;
371 assertx(!mceValue
->isStaticInPrologue());
373 // Note: if you manually CSE mceValue->methodSlot() here, gcc 4.8
374 // will strangely generate two loads instead of one.
375 if (UNLIKELY(cls
->numMethods() <= mceValue
->methodSlot())) {
376 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
378 auto const cand
= cls
->getMethod(mceValue
->methodSlot());
380 // If this class has the same func at the same method slot we're
381 // good to go. No need to recheck permissions, since we already
382 // checked them first time around.
384 // This case occurs when the current target class `cls' and the
385 // class we saw last time in mceKey have some shared ancestor that
386 // defines the method, but neither overrode the method.
387 if (LIKELY(cand
== mceValue
)) {
389 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
);
393 // If the previously called function (mceValue) was private, then
394 // the current context class must be mceValue->cls(), since we
395 // called it last time. So if the new class in `cls' derives from
396 // mceValue->cls(), its the same function that would be picked.
397 // Note that we can only get this case if there is a same-named
398 // (private or not) function deeper in the class hierarchy.
400 // In this case, we can do a fast subtype check using the classVec,
401 // because we know oldCls can't be an interface (because we observed
402 // an instance of it last time).
403 if (UNLIKELY(mceValue
->attrs() & AttrPrivate
)) {
404 auto const oldCls
= mceValue
->cls();
405 assertx(!(oldCls
->attrs() & AttrInterface
));
406 if (cls
->classVecLen() >= oldCls
->classVecLen() &&
407 cls
->classVec()[oldCls
->classVecLen() - 1] == oldCls
) {
408 // cls <: oldCls -- choose the same function as last time.
409 ar
->m_func
= mceValue
;
410 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
);
415 // If the candidate has the same name, its probably the right
416 // function. Try to prove it.
418 // We can use the invoked name `name' to compare with cand, but note
419 // that function names are case insensitive, so it's not necessarily
420 // true that mceValue->name() == name bitwise.
421 assertx(mceValue
->name()->isame(name
));
422 if (LIKELY(cand
->name() == name
)) {
423 if (LIKELY(cand
->attrs() & AttrPublic
)) {
424 // If the candidate function is public, then it has to be the
425 // right function. There can be no other function with this
426 // name on `cls', and we already ruled out the case where
427 // dispatch should've gone to a private function with the same
430 // The normal case here is an overridden public method. But this
431 // case can also occur on unrelated classes that happen to have
432 // a same-named function at the same method slot, which means we
433 // still have to check whether the new function is static.
437 if (UNLIKELY(cand
->isStaticInPrologue())) {
438 return readPublicStatic
<fatal
>(mce
, ar
, cls
, cand
);
440 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
);
444 // If the candidate function and the old function are originally
445 // declared on the same class, then we have mceKey and `cls' as
446 // related class types, and they are inheriting this (non-public)
447 // function from some shared ancestor, but have different
448 // implementations (since we already know mceValue != cand).
450 // Since the current context class could call it last time, we can
451 // call the new implementation too. We also know the new function
452 // can't be static, because the last one wasn't.
453 if (LIKELY(cand
->baseCls() == mceValue
->baseCls())) {
454 assertx(!cand
->isStaticInPrologue());
457 mce
->m_key
= reinterpret_cast<uintptr_t>(cls
);
462 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
466 void handlePrimeCacheInit(rds::Handle mce_handle
,
471 uintptr_t rawTarget
) {
472 auto const mce
= &rds::handleToRef
<Entry
>(mce_handle
);
473 if (!rds::isHandleInit(mce_handle
, rds::NormalTag
{})) {
475 mce
->m_value
= nullptr;
476 rds::initHandle(mce_handle
);
479 // If rawTarget doesn't have the flag bit we must have a smash in flight, but
480 // the call is still pointed at us. Just do a lookup.
481 if (!(rawTarget
& 0x1)) {
482 return lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
485 // We should be able to use DECLARE_FRAME_POINTER here,
486 // but that fails inside templates.
487 // Fortunately, this code is very x86 specific anyway...
488 #if defined(__x86_64__)
490 asm volatile("mov %%rbp, %0" : "=r" (framePtr
) ::);
491 #elif defined(__powerpc64__)
493 asm volatile("ld %0, 0(1)" : "=r" (framePtr
) ::);
494 #elif defined(__aarch64__)
496 asm volatile("mov %0, x29" : "=r" (framePtr
) ::);
498 ActRec
* framePtr
= ar
;
499 always_assert(false);
502 TCA callAddr
= smashableCallFromRet(TCA(framePtr
->m_savedRip
));
503 TCA movAddr
= TCA(rawTarget
>> 1);
505 // First fill the request local method cache for this call.
506 lookup
<fatal
>(mce
, ar
, name
, cls
, ctx
);
508 auto smashMov
= [&] (TCA addr
, uintptr_t value
) -> bool {
509 auto const imm
= smashableMovqImm(addr
);
510 if (!(imm
& 1)) return false;
512 smashMovq(addr
, value
);
516 // The inline cache is a 64-bit immediate, and we need to atomically
517 // set both the Func* and the Class*. We also can only cache these
518 // values if the Func* and Class* can't be deallocated, so this is
521 // - Both Func* and Class* must fit in 32-bit value (i.e. be
524 // - We must be in RepoAuthoritative mode. It is ok to cache a
525 // non-AttrPersistent class here, because if it isn't loaded in
526 // the request we'll never hit the TC fast path. But we can't
527 // do it if the Class* or Func* might be freed.
529 // - The call must not be magic or static. The code path in
530 // handleSlowPath currently assumes we've ruled this out.
532 // It's ok to store into the inline cache even if there are low bits
533 // set in mce->m_key. In that case we'll always just miss the in-TC
534 // fast path. We still need to clear the bit so handleSlowPath can
535 // tell it was smashed, though.
537 // If the situation is not cacheable, we just put a value into the
538 // immediate that will cause it to always call out to handleSlowPath.
539 auto const fval
= reinterpret_cast<uintptr_t>(mce
->m_value
);
540 auto const cval
= mce
->m_key
;
541 bool const cacheable
=
542 RuntimeOption::RepoAuthoritative
&&
543 cval
&& !(cval
& 0x3) &&
544 fval
< std::numeric_limits
<uint32_t>::max() &&
545 cval
< std::numeric_limits
<uint32_t>::max();
547 uintptr_t imm
= 0x2; /* not a Class, but clear low bit */
549 assertx(!(mce
->m_value
->attrs() & AttrStatic
));
550 imm
= fval
<< 32 | cval
;
552 if (!smashMov(movAddr
, imm
)) {
553 // Someone beat us to it. Bail early.
557 // Regardless of whether the inline cache was populated, smash the
558 // call to start doing real dispatch.
559 smashCall(callAddr
, fatal
?
560 tc::ustubs().handleSlowPathFatal
: tc::ustubs().handleSlowPath
);
564 void handlePrimeCacheInit
<false>(rds::Handle
, ActRec
*, StringData
*,
565 Class
*, Class
*, uintptr_t);
568 void handlePrimeCacheInit
<true>(rds::Handle
, ActRec
*, StringData
*,
569 Class
*, Class
*, uintptr_t);
572 void handleSlowPath
<false>(rds::Handle
, ActRec
*, StringData
*,
573 Class
*, Class
*, uintptr_t);
576 void handleSlowPath
<true>(rds::Handle
, ActRec
*, StringData
*,
577 Class
*, Class
*, uintptr_t);
579 } // namespace MethodCache
581 //=============================================================================
585 static const StringData
* mangleSmcName(const StringData
* cls
,
586 const StringData
* meth
,
588 // Implementation detail of FPushClsMethodD/F: we use "C::M:ctx" as
589 // the key for invoking static method "M" on class "C". This
590 // composes such a key. "::" is semi-arbitrary, though whatever we
591 // choose must delimit possible class and method names, so we might
592 // as well ape the source syntax
594 makeStaticString(String(cls
->data()) + String("::") +
595 String(meth
->data()) + String(":") +
599 rds::Handle
StaticMethodCache::alloc(const StringData
* clsName
,
600 const StringData
* methName
,
601 const char* ctxName
) {
602 return rds::bind
<StaticMethodCache
>(
603 rds::StaticMethod
{ mangleSmcName(clsName
, methName
, ctxName
) }
607 rds::Handle
StaticMethodFCache::alloc(const StringData
* clsName
,
608 const StringData
* methName
,
609 const char* ctxName
) {
610 return rds::bind
<StaticMethodFCache
>(
611 rds::StaticMethodF
{ mangleSmcName(clsName
, methName
, ctxName
) }
616 StaticMethodCache::lookup(rds::Handle handle
, const NamedEntity
*ne
,
617 const StringData
* clsName
,
618 const StringData
* methName
, TypedValue
* vmfp
) {
619 assertx(rds::isNormalHandle(handle
));
620 StaticMethodCache
* thiz
= static_cast<StaticMethodCache
*>
621 (handleToPtr(handle
));
622 Stats::inc(Stats::TgtCache_StaticMethodMiss
);
623 Stats::inc(Stats::TgtCache_StaticMethodHit
, -1);
624 TRACE(1, "miss %s :: %s caller %p\n",
625 clsName
->data(), methName
->data(), __builtin_return_address(0));
628 auto const cls
= Unit::loadClass(ne
, clsName
);
629 if (UNLIKELY(!cls
)) {
630 raise_error(Strings::UNKNOWN_CLASS
, clsName
->data());
634 // After this call, it's a post-condition that the RDS entry for `cls' is
635 // initialized, so make sure it has been as a side-effect of
636 // Unit::loadClass().
637 DEBUG_ONLY
auto const cls_ch
= ne
->getClassHandle();
638 assertx(rds::isHandleInit(cls_ch
));
639 assertx(rds::handleToRef
<LowPtr
<Class
>>(cls_ch
).get() == cls
);
642 LookupResult res
= lookupClsMethod(f
, cls
, methName
,
643 nullptr, // there may be an active
644 // this, but we can just fall
645 // through in that case.
646 arGetContextClass((ActRec
*)vmfp
),
648 if (LIKELY(res
== LookupResult::MethodFoundNoThis
&&
652 TRACE(1, "fill %s :: %s -> %p\n", clsName
->data(),
653 methName
->data(), f
);
654 // Do the | here instead of on every call.
655 thiz
->m_cls
= (Class
*)(uintptr_t(cls
) | 1);
657 rds::initHandle(handle
);
660 assertx(res
!= LookupResult::MethodFoundWithThis
); // Not possible: no this.
662 // Indicate to the IR that it should take even slower path
667 StaticMethodFCache::lookup(rds::Handle handle
, const Class
* cls
,
668 const StringData
* methName
, TypedValue
* vmfp
) {
670 assertx(rds::isNormalHandle(handle
));
671 StaticMethodFCache
* thiz
= static_cast<StaticMethodFCache
*>
672 (handleToPtr(handle
));
673 Stats::inc(Stats::TgtCache_StaticMethodFMiss
);
674 Stats::inc(Stats::TgtCache_StaticMethodFHit
, -1);
677 LookupResult res
= lookupClsMethod(f
, cls
, methName
,
679 arGetContextClass((ActRec
*)vmfp
),
681 assertx(res
!= LookupResult::MethodFoundWithThis
); // Not possible: no this.
682 if (LIKELY(res
== LookupResult::MethodFoundNoThis
&& !f
->isAbstract())) {
683 // We called lookupClsMethod with a NULL this and got back a method that
684 // may or may not be static. This implies that lookupClsMethod, given the
685 // same class and the same method name, will never return MagicCall*Found
686 // or MethodNotFound. It will always return the same f and if we do give it
687 // a this it will return MethodFoundWithThis iff (this->instanceof(cls) &&
688 // !f->isStatic()). this->instanceof(cls) is always true for
689 // FPushClsMethodF because it is only used for self:: and parent::
690 // calls. So, if we store f and its staticness we can handle calls with and
691 // without this completely in assembly.
694 thiz
->m_static
= f
->isStatic();
695 rds::initHandle(handle
);
696 TRACE(1, "fill staticfcache %s :: %s -> %p\n",
697 cls
->name()->data(), methName
->data(), f
);
698 Stats::inc(Stats::TgtCache_StaticMethodFFill
);
705 //////////////////////////////////////////////////////////////////////