hphp/runtime/base/rds.h

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #ifndef incl_HPHP_RUNTIME_RDS_H_
  17 #define incl_HPHP_RUNTIME_RDS_H_
  18
  19 #include <atomic>
  20 #include <cstdlib>
  21 #include <cinttypes>
  22 #include <string>
  23 #include <type_traits>
  24 #include <boost/variant.hpp>
  25 #include <folly/Range.h>
  26 #include <folly/Optional.h>
  27
  28 #include "hphp/runtime/base/types.h"
  29
  30 #include "hphp/util/alloc.h"
  31 #include "hphp/util/type-scan.h"
  32
  33 #ifndef RDS_FIXED_PERSISTENT_BASE
  34 // If RDS_FIXED_PERSISTENT_BASE is defined from compiler command line, don't
  35 // mess with it.  This makes it possible not to use fixed persistent base when
  36 // linking against jemalloc 5+.
  37 #ifndef incl_HPHP_UTIL_ALLOC_H_
  38 #error "please include alloc.h before determining RDS implementation!"
  39 #endif
  40 #if USE_JEMALLOC_EXTENT_HOOKS
  41 #define RDS_FIXED_PERSISTENT_BASE 1
  42 #endif
  43 #endif
  44
  45 namespace HPHP {
  46 struct Array;
  47 struct StringData;
  48 struct Class;
  49
  50 namespace jit {
  51 struct ArrayAccessProfile;
  52 struct ArrayIterProfile;
  53 struct ArrayKindProfile;
  54 struct CallTargetProfile;
  55 struct ClsCnsProfile;
  56 struct DecRefProfile;
  57 struct IncRefProfile;
  58 struct IsTypeStructProfile;
  59 struct MethProfile;
  60 struct SwitchProfile;
  61 struct TypeProfile;
  62 }
  63
  64 }
  65 //////////////////////////////////////////////////////////////////////
  66
  67 namespace HPHP { namespace rds {
  68
  69 //////////////////////////////////////////////////////////////////////
  70
  71 /*
  72  * The RDS (Request Data Segment) is a region of memory quickly accessible to
  73  * each hhvm thread that is running a PHP request.
  74  *
  75  * Essentially this is a per-thread memory region, along with an internal
  76  * dynamic link table to give the segment the same layout for each thread as
  77  * new data is allocated.
  78  *
  79  * The RDS starts with a small header that is statically layed out, followed by
  80  * the main "normal" segment, which is (logically) reset at the beginning of
  81  * every request. The next section, called "local", contains unshared but still
  82  * persistent data---this is data that is local to a thread but retains its
  83  * value across requests.  The final section contains shared "persistent" data,
  84  * which is data that retains the same values across requests.
  85  *
  86  * The shared persistent segment is allocated     RDS Layout:
  87  * within [1G, 4G) offset from the persistent
  88  * base (0 if RDS_FIXED_PERSISTENT_BASE is       +-------------+ <-- tl_base
  89  * defined as 1, which is safe if address from   |  Header     |
  90  * lower_malloc() is below 4G).                  +-------------+
  91  *                                               |             |
  92  * The normal region is perhaps analogous to     |  Normal     |
  93  * .bss, while the persistent region is          |    region   |
  94  * analogous to .rodata, and the local region    |             | growing higher
  95  * is similar to .data.                          +-------------+  vvv
  96  *                                               | \ \ \ \ \ \ |
  97  * When we're running in C++, the base of RDS    +-------------+  ^^^
  98  * is available via a thread local exported      |  Local      | growing lower
  99  * from this module (tl_base).  When running     |    region   |
 100  * in JIT-compiled code, a machine register      +-------------+ higher address
 101  * is reserved to always point at the base of    +-------------+
 102  * RDS.                                          | Persistent  |
 103  *                                               |     region  |
 104  *                                               +-------------+
 105  *
 106  * Every element in the "normal" segment has an associated generation number,
 107  * and the segment as a whole has a "current" generation number.  A particular
 108  * element is considered initialized if its generation number matches the
 109  * segment's current generation number.  If it does not, the element should be
 110  * considered to contain garbage and should be initialized as needed by the
 111  * element type.  Once done, it must be manually marked as initialized.
 112  *
 113  * Allocation/linking API:
 114  *
 115  *   You can allocate data from RDS in two primary ways, either by
 116  *   binding a Link, or anonymously.  The distinction is whether the
 117  *   allocated space is associated with some unique key that allows it
 118  *   to be re-found for any new attempts to allocate that symbol.
 119  *
 120  *   Anonymous allocations are created with rds::alloc.  Non-anonymous
 121  *   allocations can be created in two ways:
 122  *
 123  *     rds::bind(Symbol) uses an RDS-internal link table to find if
 124  *     there is an existing handle for the given symbol.
 125  *
 126  *     rds::Link<T>::bind allows the caller to make use of the
 127  *     uniqueness of other runtime structure (e.g. the Class
 128  *     structure) to avoid having a special key and needing to do
 129  *     lookups in the internal RDS link table.  The "key" for the
 130  *     allocation is the rds::Link<> object itself.
 131  *
 132  *   Finally, you can allocate anonymous single bits at a time with
 133  *   allocBit().
 134  *
 135  */
 136
 137 //////////////////////////////////////////////////////////////////////
 138
 139 /*
 140  * Lifetime-related hooks, exported to be called at the appropriate
 141  * times. If shouldRegister is false the resultant rds will be excluded from
 142  * the global rds list.
 143  */
 144 void requestInit();
 145 void requestExit();
 146 void threadInit(bool shouldRegister = true);
 147 void threadExit(bool shouldUnregister = true);
 148 void processInit();
 149
 150 /*
 151  * Flushing RDS means to madvise the memory away.  Should only be done
 152  * while a request is not in flight on this thread.
 153  *
 154  * This is done to conserve memory if a particular thread is unlikely
 155  * to need to serve another PHP request for a while.
 156  */
 157 void flush();
 158
 159 /*
 160  * Return the number of bytes that have been allocated from either
 161  * persistent, local, or non-persistent RDS.
 162  */
 163 size_t usedBytes();
 164 size_t usedLocalBytes();
 165 size_t usedPersistentBytes();
 166
 167 folly::Range<const char*> normalSection();
 168 folly::Range<const char*> localSection();
 169
 170 // Invoke F on each initialized allocation in the normal section. F is invoked
 171 // with a void* pointer to the data, the size of the data, and the stored
 172 // type-index.
 173 template <typename F> void forEachNormalAlloc(F);
 174 template <typename F> void forEachLocalAlloc(F);
 175
 176 /*
 177  * The thread-local pointer to the base of RDS.
 178  */
 179 extern __thread void* tl_base;
 180
 181 //////////////////////////////////////////////////////////////////////
 182
 183 /*
 184  * RDS symbols are centrally registered here.
 185  *
 186  * All StringData*'s below must be static strings.
 187  */
 188
 189 /*
 190  * Symbols for rds::Link's.
 191  */
 192 struct LinkID { const char* type; };
 193 struct LinkName { const char* type; LowStringPtr name; };
 194
 195 /*
 196  * Class constant values are TypedValue's stored in RDS.
 197  */
 198 struct ClsConstant { LowStringPtr clsName;
 199                      LowStringPtr cnsName; };
 200
 201 /*
 202  * StaticMethod{F,}Cache allocations.  These are used to cache static
 203  * method dispatch targets in a given class context.  The `name' field
 204  * here is a string that encodes the target class, property, and
 205  * source context.
 206  */
 207 struct StaticMethod  { LowStringPtr name; };
 208 struct StaticMethodF { LowStringPtr name; };
 209
 210 /*
 211  * Profiling translations may store various kinds of junk under
 212  * symbols that are keyed on translation id.  These generally should
 213  * go in Mode::Local or Mode::Persistent, depending on the use case.
 214  */
 215 #define RDS_PROFILE_SYMBOLS   \
 216   PR(ArrayAccessProfile)  \
 217   PR(ArrayIterProfile)    \
 218   PR(ArrayKindProfile)    \
 219   PR(CallTargetProfile)   \
 220   PR(ClsCnsProfile)       \
 221   PR(DecRefProfile)       \
 222   PR(IsTypeStructProfile) \
 223   PR(IncRefProfile)       \
 224   PR(MethProfile)         \
 225   PR(SwitchProfile)       \
 226   PR(TypeProfile)
 227
 228 enum class ProfileKind {
 229   None = 0,
 230 #define PR(T) T,
 231   RDS_PROFILE_SYMBOLS
 232 #undef PR
 233 };
 234
 235 struct Profile {
 236   Profile() = default;
 237
 238   Profile(TransID transId, Offset bcOff, const StringData* name)
 239     : kind{ProfileKind::None}
 240     , transId{transId}
 241     , bcOff{bcOff}
 242     , name{name}
 243   {}
 244
 245 #define PR(T) \
 246   Profile(const jit::T*, TransID transId,       \
 247           Offset bcOff, const StringData* name) \
 248     : kind{ProfileKind::T}                      \
 249     , transId{transId}                          \
 250     , bcOff{bcOff}                              \
 251     , name{name}                                \
 252   {}
 253   RDS_PROFILE_SYMBOLS
 254 #undef PR
 255
 256   ProfileKind kind;
 257   TransID transId;
 258   Offset bcOff;
 259   LowStringPtr name;
 260 };
 261
 262 /*
 263  * Static class properties in Mode::Local
 264  */
 265
 266 struct SPropCache { LowPtr<const Class> cls;
 267                     Slot slot; };
 268
 269 struct StaticMemoValue { FuncId funcId; };
 270 struct StaticMemoCache { FuncId funcId; };
 271
 272 struct LSBMemoValue {
 273   LowPtr<const Class> cls;
 274   FuncId funcId;
 275 };
 276
 277 struct LSBMemoCache {
 278   LowPtr<const Class> cls;
 279   FuncId funcId;
 280 };
 281
 282 struct TSCache {
 283   FuncId funcId;
 284 };
 285
 286
 287 using Symbol = boost::variant<
 288   LinkName,
 289   LinkID,
 290   ClsConstant,
 291   StaticMethod,
 292   StaticMethodF,
 293   Profile,
 294   SPropCache,
 295   StaticMemoValue,
 296   StaticMemoCache,
 297   LSBMemoValue,
 298   LSBMemoCache,
 299   TSCache
 300 >;
 301
 302 //////////////////////////////////////////////////////////////////////
 303
 304 enum class Mode : unsigned {
 305   Normal        = 1u << 0,
 306   Local         = 1u << 1,
 307   Persistent    = 1u << 2,
 308
 309   NonPersistent = Normal | Local,
 310   NonLocal      = Normal | Persistent,
 311   NonNormal     = Local  | Persistent,
 312
 313   Any           = Normal | Local | Persistent,
 314 };
 315
 316 using ModeU = std::underlying_type<Mode>::type;
 317
 318 template<Mode Mask> constexpr bool maybe(Mode mode) {
 319   return !!(static_cast<ModeU>(mode) & static_cast<ModeU>(Mask));
 320 }
 321
 322 template<Mode RHS> constexpr bool in(Mode LHS) {
 323   return !(static_cast<ModeU>(LHS) & ~static_cast<ModeU>(RHS));
 324 }
 325
 326 constexpr bool pure(Mode mask) {
 327   return !(static_cast<ModeU>(mask) & (static_cast<ModeU>(mask) - 1));
 328 }
 329
 330 /*
 331  * Handles into Request Data Segment.
 332  *
 333  * For `Normal` and `Local` mode, it is an offset from rds::tl_base. The offset
 334  * must be smaller than 1 << 30.
 335  *
 336  * For `Persistent` mode, the handle is an offset from `s_persistent_base`.
 337  * When `RDS_FIXED_PERSISTENT_BASE` is defined, `s_persistent_base` is always 0,
 338  * and the handle is the same as the address, and must be at least 1 << 30.
 339  */
 340 using Handle = uint32_t;
 341 constexpr Handle kUninitHandle = 0;
 342 constexpr Handle kBeingBound = 1;
 343 constexpr Handle kBeingBoundWithWaiters = 2;
 344 constexpr Handle kMinPersistentHandle = 1u << 30;
 345 constexpr Handle kMaxHandle = std::numeric_limits<Handle>::max();
 346
 347 /*
 348  * Normal segment element generation numbers.
 349  *
 350  * The segment's current generation number will never be kInvalidGenNumber;
 351  * thus, it is safe to use to mark an element as being uninitialized
 352  * unconditionally.
 353  */
 354 using GenNumber = uint8_t;
 355 constexpr GenNumber kInvalidGenNumber = 0;
 356
 357 /*
 358  * Tag passed to RDS handle initialization routines to indicate that the handle
 359  * is known to be normal.
 360  */
 361 enum class NormalTag {};
 362
 363 /*
 364  * rds::Link<T,M> is a thin, typed wrapper around an rds::Handle.
 365  *
 366  * Note that nothing prevents using non-POD types with this.  But nothing here
 367  * is going to run the constructor.  (In the non-persistent region, the space
 368  * for T will contain garbage at the start of each request.)
 369  *
 370  * Links are atomic types.  All apis may be called concurrently by
 371  * multiple threads, and the alloc() api guarantees only a single
 372  * caller will actually allocate new space in RDS.
 373  */
 374 template<class T, Mode M>
 375 struct Link {
 376   explicit Link(Handle handle = kUninitHandle);
 377
 378   /*
 379    * We allow copy construction or assignment from a Link of a narrower Mode,
 380    * but it is a compile-time error if we try to do it the other way.
 381    */
 382   Link(const Link<T,M>&);
 383   Link<T,M>& operator=(const Link<T,M>&);
 384
 385   template<Mode OM> /* implicit */ Link(
 386       const Link<T,OM>&,
 387       typename std::enable_if<in<M>(OM), bool>::type = false);
 388
 389   template<Mode OM> typename std::enable_if<in<M>(OM),Link<T,M>>::type&
 390   operator=(const Link<T,OM>&);
 391
 392   /*
 393    * Ensure this Link is bound to an RDS allocation.
 394    *
 395    * Allocation is atomic and idempotent.  This ensures that only one thread
 396    * allocates the handle, and (in the cast of persistent handles) that the
 397    * value is initialized as part of the operation.  The effect is that other
 398    * threads only ever see an unbound handle or a bound handle with a valid
 399    * value.
 400    *
 401    * rds::Link's are "keyed" by context---generally meaning that they're owned
 402    * by an object with an independent means of enforcing uniqueness.  The `sym`
 403    * argument should also be unique, but is used only for optimizations.
 404    *
 405    * If the Link is already bound, both `mode` and `Align` are ignored.  If the
 406    * mode is not persistent, `init_val` is ignored.
 407    *
 408    * Post: bound()
 409    */
 410   template<size_t Align = alignof(T)>
 411   void bind(Mode mode, Symbol sym, const T* init_val = nullptr);
 412
 413   /*
 414    * Dereference a Link and access its RDS memory for the current thread.
 415    *
 416    * Pre: bound()
 417    */
 418   T& operator*() const;
 419   T* operator->() const;
 420   T* get() const;
 421
 422   /*
 423    * Whether this Link is bound to RDS memory or not (i.e., whether its
 424    * internal handle is valid).
 425    */
 426   bool bound() const;
 427
 428   /*
 429    * Access to the underlying rds::Handle.
 430    */
 431   Handle handle() const;
 432
 433   /*
 434    * Access to the underlying rds::Handle; returns kUninitHandle if
 435    * its not bound.
 436    */
 437   Handle maybeHandle() const;
 438
 439   /*
 440    * Return the generation number of this element.
 441    *
 442    * Pre: bound() && isNormal()
 443    */
 444   GenNumber genNumber() const;
 445
 446   /*
 447    * Return the handle for this element's generation number.
 448    *
 449    * Pre: bound() && isNormal()
 450    */
 451   Handle genNumberHandle() const;
 452
 453   /*
 454    * Check whether this element is initialized.
 455    *
 456    * This is only an interesting designation for normal handles, where we need
 457    * to check the generation number.  Persistent handles are expected not to be
 458    * "published" until they are in some sort of initialized state (which might
 459    * simply be some nullish sentinel value).
 460    *
 461    * Pre: bound()
 462    */
 463   bool isInit() const;
 464
 465   /*
 466    * Manually mark this element as initialized or uninitialized.
 467    *
 468    * Pre: bound() && isNormal()
 469    */
 470   void markInit() const;
 471   void markUninit() const;
 472
 473   /*
 474    * Initialize this element to `val'.
 475    *
 476    * Anything previously stored in the element is considered to be garbage, so
 477    * it is not destructed.  initWith() can thus be used to unconditionally
 478    * initialize something that might already be inited, but only if it's
 479    * trivially destructible.
 480    *
 481    * Post: isInit()
 482    */
 483   void initWith(const T& val) const;
 484   void initWith(T&& val) const;
 485
 486   /*
 487    * Check which segment this element resides in.
 488    *
 489    * Pre: bound()
 490    */
 491   bool isNormal() const;
 492   bool isLocal() const;
 493   bool isPersistent() const;
 494
 495   /*
 496    * For access from the JIT only.
 497    */
 498   static constexpr size_t handleOff() {
 499     return offsetof(Link, m_handle);
 500   }
 501
 502 private:
 503   template<class OT, Mode OM> friend struct Link;
 504
 505   void checkSanity();
 506
 507   Handle raw() const { return m_handle.load(std::memory_order_relaxed); }
 508   std::atomic<Handle> m_handle;
 509 };
 510
 511 /*
 512  * Return a bound link to memory from RDS, using the given Symbol.
 513  *
 514  * Mode indicates whether the memory should be placed in the persistent region
 515  * or not, Align indicates the alignment requirements, and extraSize allows for
 516  * allocating additional space beyond sizeof(T), for variable-length structures
 517  * (not allowed for normal mode).  All three arguments are ignored if there is
 518  * already an allocation for the Symbol---they only affect the first caller for
 519  * the given Symbol.
 520  *
 521  * N indicates that the binding for `key' will always be in the "normal" RDS
 522  * region; it is allowed to be true only if `key' is only ever bound with
 523  * Mode::Normal.
 524  */
 525 template<class T, Mode M, size_t Align = alignof(T)>
 526 Link<T,M> bind(Symbol key, size_t extraSize = 0);
 527
 528 /*
 529  * Remove a bound link from RDS metadata. The actual space in RDS is
 530  * not reclaimed.
 531  */
 532 void unbind(Symbol, Handle);
 533
 534 /*
 535  * Try to bind to a symbol in RDS, returning an unbound link if the
 536  * Symbol isn't already allocated in RDS.  Mode and alignment are not
 537  * specified---if the symbol is already bound, these are already
 538  * fixed.  The `T' must be the same `T' that the symbol is mapped to,
 539  * if it's already mapped.
 540  */
 541 template<class T, Mode M = Mode::Any>
 542 Link<T,M> attach(Symbol key);
 543
 544 /*
 545  * Allocate anonymous memory from RDS.
 546  *
 547  * The memory is not keyed on any Symbol, so the handle in the returned Link
 548  * will be unique.
 549  */
 550 template<class T, Mode M = Mode::Normal, size_t Align = alignof(T)>
 551 Link<T,M> alloc();
 552
 553 /*
 554  * Allocate a single anonymous bit from non-persistent RDS.  The bit
 555  * can be manipulated with testAndSetBit().
 556  *
 557  * Note: the returned integer is *not* an rds::Handle.
 558  */
 559 size_t allocBit();
 560 bool testAndSetBit(size_t bit);
 561
 562 folly::Optional<Symbol> reverseLink(Handle handle);
 563
 564 //////////////////////////////////////////////////////////////////////
 565
 566 /*
 567  * Retrieve the current generation number for the normal segment.
 568  */
 569 GenNumber currentGenNumber();
 570
 571 /*
 572  * Retrieve the handle for the current generation number for the normal segment.
 573  */
 574 Handle currentGenNumberHandle();
 575
 576 //////////////////////////////////////////////////////////////////////
 577
 578 /*
 579  * Dereference an un-typed rds::Handle which is guaranteed to be in one of the
 580  * `modes`, optionally specifying a specific RDS base to use.
 581  */
 582 template<class T, Mode M> T& handleToRef(Handle h);
 583 template<class T, Mode M> T& handleToRef(void* base, Handle h);
 584
 585 /*
 586  * Conversion between a pointer and an rds::Handle which is guaranteed to be in
 587  * one of the modes specified in `M`.
 588  */
 589 template<class T = void, Mode M> T* handleToPtr(Handle h);
 590 template<Mode M> Handle ptrToHandle(const void* ptr);
 591 template<Mode M> Handle ptrToHandle(uintptr_t ptr);
 592
 593 /*
 594  * Whether `handle' looks valid---i.e., whether it lies within the RDS bounds.
 595  */
 596 bool isValidHandle(Handle handle);
 597
 598 /*
 599  * Whether `handle' is from the normal RDS region.
 600  *
 601  * Pre: isValidHandle(handle)
 602  */
 603 bool isNormalHandle(Handle handle);
 604
 605 /*
 606  * Whether `handle' is from the local RDS region.
 607  *
 608  * Pre: isValidHandle(handle)
 609  */
 610 bool isLocalHandle(Handle handle);
 611
 612 /*
 613  * Whether `handle' is from the persistent RDS region.
 614  *
 615  * Pre: isValidHandle(handle)
 616  */
 617 bool isPersistentHandle(Handle handle);
 618
 619 /*
 620  * The generation number associated with `handle'.
 621  *
 622  * Pre: isNormalHandle(handle)
 623  */
 624 GenNumber genNumberOf(Handle handle);
 625
 626 /*
 627  * The handle for the generation number associated with `handle'.
 628  *
 629  * Pre: isNormalHandle(handle)
 630  */
 631 Handle genNumberHandleFrom(Handle handle);
 632
 633 /*
 634  * Whether the handle has been bound.
 635  */
 636 bool isHandleBound(Handle handle);
 637
 638 /*
 639  * Whether the element associated with `handle' is initialized.
 640  */
 641 bool isHandleInit(Handle handle);
 642 bool isHandleInit(Handle handle, NormalTag);
 643
 644 /*
 645  * Mark the element associated with `handle' as being initialized.
 646  *
 647  * Pre: isNormalHandle(handle)
 648  */
 649 void initHandle(Handle handle);
 650
 651 /*
 652  * Marks the element associated with the supplied handle as being
 653  * uninitialized. This happens automatically after every request, but can be
 654  * done manually with this.
 655  *
 656  * Pre: isNormalHandle(handle)
 657  */
 658 void uninitHandle(Handle handle);
 659
 660 /*
 661  * Used to record information about the rds handle h in the
 662  * perf-data-pid.map (if enabled).
 663  * The type indicates the type of entry (eg ClsConstant), and the
 664  * msg identifies this particular entry (eg function-name:local-name)
 665  */
 666 void recordRds(Handle h, size_t size,
 667                folly::StringPiece type, folly::StringPiece msg);
 668 void recordRds(Handle h, size_t size, const Symbol& sym);
 669
 670 void visitSymbols(std::function<void(const Symbol&,Handle,uint32_t)> fun);
 671
 672 /*
 673  * Return a list of all the tl_bases for any threads that are using RDS
 674  */
 675 std::vector<void*> allTLBases();
 676
 677 //////////////////////////////////////////////////////////////////////
 678
 679 extern rds::Link<bool, Mode::Persistent> s_persistentTrue;
 680
 681 }}
 682
 683 #include "hphp/runtime/base/rds-inl.h"
 684
 685 #endif