hphp/runtime/base/rds.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/base/rds.h"
  17
  18 #include <cassert>
  19 #include <cstdio>
  20 #include <execinfo.h>
  21
  22 #include <sys/mman.h>
  23 #include <atomic>
  24
  25 #include "folly/String.h"
  26 #include "folly/Hash.h"
  27 #include "folly/Bits.h"
  28
  29 #include "hphp/util/maphuge.h"
  30 #include "hphp/util/mutex.h"
  31 #include "hphp/util/lock.h"
  32
  33 #include "hphp/runtime/base/complex-types.h"
  34 #include "hphp/runtime/vm/debug/debug.h"
  35
  36 namespace HPHP { namespace RDS {
  37
  38 //////////////////////////////////////////////////////////////////////
  39
  40 namespace {
  41
  42 //////////////////////////////////////////////////////////////////////
  43
  44 // Current allocation frontier for the non-persistent region.
  45 size_t s_frontier = sizeof(Header);
  46
  47 // Frontier and base of the persistent region.
  48 size_t s_persistent_base = 0;
  49 size_t s_persistent_frontier = 0;
  50
  51 /*
  52  * This mutex protects actually allocating from RDS (the above
  53  * statics).  It is ordered *after* the locks in s_linkTable.
  54  */
  55 SimpleMutex s_allocMutex(false /*recursive*/, RankLeaf);
  56
  57 //////////////////////////////////////////////////////////////////////
  58
  59 struct SymbolKind : boost::static_visitor<std::string> {
  60   std::string operator()(StaticLocal k) const { return "StaticLocal"; }
  61   std::string operator()(ClsConstant k) const { return "ClsConstant"; }
  62   std::string operator()(StaticProp k) const { return "StaticProp"; }
  63   std::string operator()(StaticMethod k) const { return "StaticMethod"; }
  64   std::string operator()(StaticMethodF k) const { return "StaticMethodF"; }
  65 };
  66
  67 struct SymbolRep : boost::static_visitor<std::string> {
  68   std::string operator()(StaticLocal k) const {
  69     const Func* func = Func::fromFuncId(k.funcId);
  70     const Class* cls = getOwningClassForFunc(func);
  71     std::string name;
  72     if (cls != func->cls()) {
  73       name = cls->name()->toCppString() + "::" +
  74         func->name()->toCppString();
  75     } else {
  76       name = func->fullName()->toCppString();
  77     }
  78     return name + "::" + k.name->toCppString();
  79   }
  80
  81   std::string operator()(ClsConstant k) const {
  82     return k.clsName->data() + std::string("::") + k.cnsName->data();
  83   }
  84
  85   std::string operator()(StaticProp k)    const { return k.name->data(); }
  86   std::string operator()(StaticMethod k)  const { return k.name->data(); }
  87   std::string operator()(StaticMethodF k) const { return k.name->data(); }
  88 };
  89
  90 struct SymbolEq : boost::static_visitor<bool> {
  91   template<class T, class U>
  92   typename std::enable_if<
  93     !std::is_same<T,U>::value,
  94     bool
  95   >::type operator()(const T&, const U&) const { return false; }
  96
  97   bool operator()(StaticLocal k1, StaticLocal k2) const {
  98     assert(k1.name->isStatic() && k2.name->isStatic());
  99     return k1.funcId == k2.funcId && k1.name == k2.name;
 100   }
 101
 102   bool operator()(ClsConstant k1, ClsConstant k2) const {
 103     assert(k1.clsName->isStatic() && k1.cnsName->isStatic());
 104     assert(k2.clsName->isStatic() && k2.cnsName->isStatic());
 105     return k1.clsName->isame(k2.clsName) &&
 106            k1.cnsName == k2.cnsName;
 107   }
 108
 109   bool operator()(StaticProp k1, StaticProp k2) const {
 110     assert(k1.name->isStatic() && k2.name->isStatic());
 111     return k1.name == k2.name;
 112   }
 113
 114   template<class T>
 115   typename std::enable_if<
 116     std::is_same<T,StaticMethod>::value ||
 117       std::is_same<T,StaticMethodF>::value,
 118     bool
 119   >::type operator()(const T& t1, const T& t2) const {
 120     assert(t1.name->isStatic() && t2.name->isStatic());
 121     return t1.name->isame(t2.name);
 122   }
 123 };
 124
 125 struct SymbolHash : boost::static_visitor<size_t> {
 126   size_t operator()(StaticLocal k) const {
 127     return folly::hash::hash_128_to_64(
 128       std::hash<FuncId>()(k.funcId),
 129       k.name->hash()
 130     );
 131   }
 132
 133   size_t operator()(ClsConstant k) const {
 134     return folly::hash::hash_128_to_64(
 135       k.clsName->hash(),
 136       k.cnsName->hash()
 137     );
 138   }
 139
 140   size_t operator()(StaticProp k)    const { return k.name->hash(); }
 141   size_t operator()(StaticMethod k)  const { return k.name->hash(); }
 142   size_t operator()(StaticMethodF k) const { return k.name->hash(); }
 143 };
 144
 145 struct HashCompare {
 146   bool equal(const Symbol& k1, const Symbol& k2) const {
 147     return boost::apply_visitor(SymbolEq(), k1, k2);
 148   }
 149
 150   size_t hash(const Symbol& k) const {
 151     return boost::apply_visitor(SymbolHash(), k);
 152   }
 153 };
 154
 155 typedef tbb::concurrent_hash_map<
 156   Symbol,
 157   Handle,
 158   HashCompare
 159 > LinkTable;
 160
 161 LinkTable s_linkTable;
 162
 163 //////////////////////////////////////////////////////////////////////
 164
 165 }
 166
 167 //////////////////////////////////////////////////////////////////////
 168
 169 namespace detail {
 170
 171 Handle alloc(Mode mode, size_t numBytes, size_t align) {
 172   s_allocMutex.assertOwnedBySelf();
 173   align = folly::nextPowTwo(align);
 174   auto& frontier = mode == Mode::Persistent ? s_persistent_frontier
 175                                             : s_frontier;
 176
 177   // Note: it's ok not to zero new allocations, because we've never
 178   // done anything with this part of the page yet, so it must still be
 179   // zero.
 180   frontier += align - 1;
 181   frontier &= ~(align - 1);
 182   frontier += numBytes;
 183
 184   auto const limit = mode == Mode::Persistent
 185     ? RuntimeOption::EvalJitTargetCacheSize
 186     : s_persistent_base;
 187   always_assert(frontier < limit);
 188
 189   return frontier - numBytes;
 190 }
 191
 192 Handle allocUnlocked(Mode mode, size_t numBytes, size_t align) {
 193   SimpleLock l(s_allocMutex);
 194   return alloc(mode, numBytes, align);
 195 }
 196
 197 Handle bindImpl(Symbol key, Mode mode, size_t sizeBytes, size_t align) {
 198   LinkTable::const_accessor acc;
 199   if (s_linkTable.find(acc, key)) return acc->second;
 200
 201   SimpleLock l(s_allocMutex);
 202   if (s_linkTable.find(acc, key)) return acc->second;
 203
 204   auto const retval = alloc(mode, sizeBytes, align);
 205
 206   recordRds(retval, sizeBytes, key);
 207   if (!s_linkTable.insert(LinkTable::value_type(key, retval))) {
 208     always_assert(0);
 209   }
 210   return retval;
 211 }
 212
 213 void bindOnLinkImpl(std::atomic<Handle>& handle,
 214                     Mode mode,
 215                     size_t sizeBytes,
 216                     size_t align) {
 217   SimpleLock l(s_allocMutex);
 218   if (handle.load(std::memory_order_relaxed) == kInvalidHandle) {
 219     handle.store(alloc(mode, sizeBytes, align), std::memory_order_relaxed);
 220   }
 221 }
 222
 223 }
 224
 225
 226 //////////////////////////////////////////////////////////////////////
 227
 228 __thread void* tl_base = nullptr;
 229 static __thread std::aligned_storage<
 230   sizeof(Array),
 231   alignof(Array)
 232 >::type s_constantsStorage;
 233
 234 //////////////////////////////////////////////////////////////////////
 235
 236 static size_t s_next_bit;
 237 static size_t s_bits_to_go;
 238 static int s_tc_fd;
 239
 240 // Mapping from names to targetcache locations.
 241 typedef tbb::concurrent_hash_map<const StringData*, Handle,
 242         StringDataHashICompare>
 243   HandleMapIS;
 244
 245 typedef tbb::concurrent_hash_map<const StringData*, Handle,
 246         StringDataHashCompare>
 247   HandleMapCS;
 248
 249 //////////////////////////////////////////////////////////////////////
 250
 251 void requestInit() {
 252   assert(tl_base);
 253   new (&s_constantsStorage) Array();
 254   assert(!s_constants().get());
 255   memset(tl_base, 0, s_frontier);
 256 }
 257
 258 void requestExit() {
 259   s_constants().detach(); // it will be swept
 260   // Don't bother running the dtor ...
 261 }
 262
 263 void flush() {
 264   if (madvise(tl_base, s_frontier, MADV_DONTNEED) == -1) {
 265     fprintf(stderr, "RDS madvise failure: %s\n",
 266       folly::errnoStr(errno).c_str());
 267   }
 268 }
 269
 270 size_t usedBytes() {
 271   return s_frontier;
 272 }
 273
 274 size_t usedPersistentBytes() {
 275   return s_persistent_frontier - s_persistent_base;
 276 }
 277
 278 Array& s_constants() {
 279   void* vp = &s_constantsStorage;
 280   return *static_cast<Array*>(vp);
 281 }
 282
 283 //////////////////////////////////////////////////////////////////////
 284
 285 size_t allocBit() {
 286   SimpleLock l(s_allocMutex);
 287   if (!s_bits_to_go) {
 288     static const int kNumBytes = 512;
 289     static const int kNumBytesMask = kNumBytes - 1;
 290     s_next_bit = s_frontier * CHAR_BIT;
 291     // allocate at least kNumBytes bytes, and make sure we end
 292     // on a 64 byte aligned boundary.
 293     int bytes = ((~s_frontier + 1) & kNumBytesMask) + kNumBytes;
 294     s_bits_to_go = bytes * CHAR_BIT;
 295     s_frontier += bytes;
 296     recordRds(s_frontier - bytes, bytes, "Unknown", "bits");
 297   }
 298   s_bits_to_go--;
 299   return s_next_bit++;
 300 }
 301
 302 bool testAndSetBit(size_t bit) {
 303   Handle handle = bit / CHAR_BIT;
 304   unsigned char mask = 1 << (bit % CHAR_BIT);
 305   bool ret = handleToRef<unsigned char>(handle) & mask;
 306   handleToRef<unsigned char>(handle) |= mask;
 307   return ret;
 308 }
 309
 310 bool isPersistentHandle(Handle handle) {
 311   assert(handle >= 0 && handle < RuntimeOption::EvalJitTargetCacheSize);
 312   return handle >= (unsigned)s_persistent_base;
 313 }
 314
 315 static void initPersistentCache() {
 316   SimpleLock l(s_allocMutex);
 317   if (s_tc_fd) return;
 318   char tmpName[] = "/tmp/tcXXXXXX";
 319   s_tc_fd = mkstemp(tmpName);
 320   always_assert(s_tc_fd != -1);
 321   unlink(tmpName);
 322   s_persistent_base = RuntimeOption::EvalJitTargetCacheSize * 3 / 4;
 323   s_persistent_base -= s_persistent_base & (4 * 1024 - 1);
 324   ftruncate(s_tc_fd,
 325             RuntimeOption::EvalJitTargetCacheSize - s_persistent_base);
 326   s_persistent_frontier = s_persistent_base;
 327 }
 328
 329 void threadInit() {
 330   if (!s_tc_fd) {
 331     initPersistentCache();
 332   }
 333
 334   tl_base = mmap(nullptr, RuntimeOption::EvalJitTargetCacheSize,
 335                  PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
 336   always_assert(tl_base != MAP_FAILED);
 337   numa_bind_to(tl_base, s_persistent_base, s_numaNode);
 338   if (RuntimeOption::EvalMapTgtCacheHuge) {
 339     hintHuge(tl_base, RuntimeOption::EvalJitTargetCacheSize);
 340   }
 341
 342   void *shared_base = (char*)tl_base + s_persistent_base;
 343   /*
 344    * map the upper portion of the RDS to a shared area This is used
 345    * for persistent classes and functions, so they are always defined,
 346    * and always visible to all threads.
 347    */
 348   void *mem = mmap(shared_base,
 349                    RuntimeOption::EvalJitTargetCacheSize - s_persistent_base,
 350                    PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, s_tc_fd, 0);
 351   always_assert(mem == shared_base);
 352
 353   if (RuntimeOption::EvalPerfDataMap) {
 354     Debug::DebugInfo::recordDataMap(
 355       tl_base,
 356       (char*)tl_base + RuntimeOption::EvalJitTargetCacheSize,
 357       "rds");
 358   }
 359 }
 360
 361 void threadExit() {
 362   if (RuntimeOption::EvalPerfDataMap) {
 363     Debug::DebugInfo::recordDataMap(
 364       tl_base,
 365       (char*)tl_base + RuntimeOption::EvalJitTargetCacheSize,
 366       "-rds");
 367   }
 368   munmap(tl_base, RuntimeOption::EvalJitTargetCacheSize);
 369 }
 370
 371 void recordRds(Handle h, size_t size,
 372                const std::string& type, const std::string& msg) {
 373   if (RuntimeOption::EvalPerfDataMap) {
 374     Debug::DebugInfo::recordDataMap(
 375       (char*)(intptr_t)h,
 376       (char*)(intptr_t)h + size,
 377       folly::format("rds+{}-{}", type, msg).str());
 378   }
 379 }
 380
 381 void recordRds(Handle h, size_t size, const Symbol& sym) {
 382   if (RuntimeOption::EvalPerfDataMap) {
 383     recordRds(h, size,
 384               boost::apply_visitor(SymbolKind(), sym),
 385               boost::apply_visitor(SymbolRep(), sym));
 386   }
 387 }
 388
 389 //////////////////////////////////////////////////////////////////////
 390
 391 }}