hphp/runtime/vm/jit/code-cache.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/code-cache.h"
  18
  19 #include "hphp/runtime/vm/func.h"
  20 #include "hphp/runtime/vm/jit/mcgen.h"
  21 #include "hphp/runtime/vm/jit/tc.h"
  22 #include "hphp/runtime/vm/jit/translator.h"
  23
  24 #include "hphp/runtime/base/program-functions.h"
  25
  26 #include "hphp/util/alloc.h"
  27 #include "hphp/util/asm-x64.h"
  28 #include "hphp/util/numa.h"
  29 #include "hphp/util/trace.h"
  30
  31 namespace HPHP { namespace jit {
  32
  33 TRACE_SET_MOD(mcg);
  34
  35 // This value should be enough bytes to emit a REQ_RETRANSLATE: lea (4 or 7
  36 // bytes), movq (10 bytes), and jmp (5 bytes). We then add some extra slack for
  37 // safety.
  38 static const int kMinTranslationBytes = 32;
  39
  40 /* Initialized by RuntimeOption. */
  41 uint64_t CodeCache::AHotSize = 0;
  42 uint64_t CodeCache::ASize = 0;
  43 uint64_t CodeCache::AProfSize = 0;
  44 uint64_t CodeCache::AColdSize = 0;
  45 uint64_t CodeCache::AFrozenSize = 0;
  46 uint64_t CodeCache::GlobalDataSize = 0;
  47 uint64_t CodeCache::AMaxUsage = 0;
  48 uint64_t CodeCache::AColdMaxUsage = 0;
  49 uint64_t CodeCache::AFrozenMaxUsage = 0;
  50 bool CodeCache::MapTCHuge = false;
  51 uint32_t CodeCache::AutoTCShift = 0;
  52 uint32_t CodeCache::TCNumHugeHotMB = 0;
  53 uint32_t CodeCache::TCNumHugeColdMB = 0;
  54
  55 CodeCache::CodeCache()
  56   : m_useHot{RuntimeOption::RepoAuthoritative && CodeCache::AHotSize > 0}
  57 {
  58   static const size_t kRoundUp = 2 << 20;
  59
  60   auto ru = [=] (size_t sz) { return sz + (-sz & (kRoundUp - 1)); };
  61   auto rd = [=] (size_t sz) { return sz & ~(kRoundUp - 1); };
  62
  63   // We want to ensure that all code blocks are close to each other so that we
  64   // can short jump/point between them. Thus we allocate one slab and divide it
  65   // between the various blocks.
  66   auto const thread_local_size = ru(
  67     RuntimeOption::EvalThreadTCMainBufferSize +
  68     RuntimeOption::EvalThreadTCColdBufferSize +
  69     RuntimeOption::EvalThreadTCFrozenBufferSize +
  70     RuntimeOption::EvalThreadTCDataBufferSize
  71   );
  72
  73   auto const kAHotSize    = ru(CodeCache::AHotSize);
  74   auto const kASize       = ru(CodeCache::ASize);
  75   auto const kAProfSize   = ru(CodeCache::AProfSize);
  76   auto const kAColdSize   = ru(CodeCache::AColdSize);
  77   auto const kAFrozenSize = ru(CodeCache::AFrozenSize);
  78
  79   auto kGDataSize = ru(CodeCache::GlobalDataSize);
  80   m_totalSize = kAHotSize + kASize + kAColdSize + kAProfSize +
  81                 kAFrozenSize + kGDataSize + thread_local_size;
  82   m_codeSize = m_totalSize - kGDataSize;
  83
  84   if ((kASize < (10 << 20)) ||
  85       (kAColdSize < (4 << 20)) ||
  86       (kAFrozenSize < (6 << 20)) ||
  87       (kGDataSize < (2 << 20))) {
  88     fprintf(stderr, "Allocation sizes ASize, AColdSize, AFrozenSize and "
  89                     "GlobalDataSize are too small.\n");
  90     exit(1);
  91   }
  92
  93   if (m_totalSize > (2ul << 30)) {
  94     fprintf(stderr,"Combined size of ASize, AColdSize, AFrozenSize and "
  95                     "GlobalDataSize must be < 2GiB to support 32-bit relative "
  96                     "addresses\n");
  97     exit(1);
  98   }
  99
 100   auto enhugen = [&](void* base, int numMB) {
 101     if (CodeCache::MapTCHuge) {
 102       assertx((uintptr_t(base) & (kRoundUp - 1)) == 0);
 103       hintHugeDeleteData((char*)base, numMB << 20,
 104                          PROT_READ | PROT_WRITE | PROT_EXEC,
 105                          false /* MAP_SHARED */);
 106     }
 107   };
 108
 109   // We want to ensure that all code blocks are close to each other so that we
 110   // can short jump/point between them. Thus we allocate one slab and divide it
 111   // between the various blocks.
 112
 113   // Using sbrk to ensure its in the bottom 2G, so we avoid the need for
 114   // trampolines, and get to use shorter instructions for tc addresses.
 115   size_t allocationSize = m_totalSize;
 116   size_t baseAdjustment = 0;
 117   uint8_t* base = (uint8_t*)sbrk(0);
 118
 119   // Adjust the start of TC relative to hot runtime code. What really matters
 120   // is a number of 2MB pages in-between. We appear to benefit from odd numbers.
 121   auto const shiftTC = [&]() -> size_t {
 122     if (!CodeCache::AutoTCShift || __hot_start == nullptr) return 0;
 123     // Make sure the offset from hot text is either odd or even number
 124     // of huge pages.
 125     const auto hugePagesDelta = (ru(reinterpret_cast<size_t>(base)) -
 126                                  rd(reinterpret_cast<size_t>(__hot_start))) /
 127                                 kRoundUp;
 128     return ((hugePagesDelta & 1) == (CodeCache::AutoTCShift & 1))
 129       ? 0
 130       : kRoundUp;
 131   };
 132
 133   if (base != (uint8_t*)-1) {
 134     assertx(!(allocationSize & (kRoundUp - 1)));
 135     // Make sure that we have space to round up to the start of a huge page
 136     allocationSize += -(uint64_t)base & (kRoundUp - 1);
 137     allocationSize += shiftTC();
 138     base = (uint8_t*)sbrk(allocationSize);
 139     baseAdjustment = allocationSize - m_totalSize;
 140   }
 141   if (base == (uint8_t*)-1) {
 142     allocationSize = m_totalSize + kRoundUp - 1;
 143     if (CodeCache::AutoTCShift) {
 144       allocationSize += kRoundUp;
 145     }
 146     base = (uint8_t*)low_malloc(allocationSize);
 147     if (!base) {
 148       base = (uint8_t*)malloc(allocationSize);
 149     }
 150     if (!base) {
 151       fprintf(stderr, "could not allocate %zd bytes for translation cache\n",
 152               allocationSize);
 153       exit(1);
 154     }
 155     baseAdjustment = -(uint64_t)base & (kRoundUp - 1);
 156     baseAdjustment += shiftTC();
 157   } else {
 158     low_malloc_skip_huge(base, base + allocationSize - 1);
 159   }
 160   assertx(base);
 161   base += baseAdjustment;
 162
 163   m_base = base;
 164
 165   numa_interleave(base, m_totalSize);
 166
 167   if (kAHotSize) {
 168     TRACE(1, "init ahot @%p\n", base);
 169     m_hot.init(base, kAHotSize, "hot");
 170     enhugen(base, kAHotSize >> 20);
 171     base += kAHotSize;
 172   }
 173
 174   TRACE(1, "init a @%p\n", base);
 175
 176   m_main.init(base, kASize, "main");
 177   enhugen(base, CodeCache::TCNumHugeHotMB);
 178   base += kASize;
 179
 180   TRACE(1, "init aprof @%p\n", base);
 181   m_prof.init(base, kAProfSize, "prof");
 182   base += kAProfSize;
 183
 184   TRACE(1, "init acold @%p\n", base);
 185   m_cold.init(base, kAColdSize, "cold");
 186   enhugen(base, CodeCache::TCNumHugeColdMB);
 187   base += kAColdSize;
 188
 189   TRACE(1, "init thread_local @%p\n", base);
 190   m_threadLocalStart = base;
 191   base += thread_local_size;
 192
 193   TRACE(1, "init afrozen @%p\n", base);
 194   m_frozen.init(base, kAFrozenSize, "afrozen");
 195   base += kAFrozenSize;
 196
 197   TRACE(1, "init gdata @%p\n", base);
 198   m_data.init(base, kGDataSize, "gdata");
 199   base += kGDataSize;
 200
 201   // The default on linux for the newly allocated memory is read/write/exec
 202   // but on some systems its just read/write. Call unprotect to ensure that
 203   // the memory is marked executable.
 204   unprotect();
 205
 206   // Assert that no one is actually writing to or reading from the pseudo
 207   // addresses used to emit thread local translations
 208   if (thread_local_size) {
 209     mprotect(m_threadLocalStart, thread_local_size, PROT_NONE);
 210   }
 211   m_threadLocalSize = thread_local_size;
 212
 213   assertx(base - m_base <= allocationSize);
 214   assertx(base - m_base + 2 * kRoundUp > allocationSize);
 215   assertx(base - m_base <= (2ul << 30));
 216 }
 217
 218 CodeBlock& CodeCache::blockFor(CodeAddress addr) {
 219   return codeBlockChoose(addr, m_main, m_hot, m_prof, m_cold, m_frozen);
 220 }
 221
 222 const CodeBlock& CodeCache::blockFor(CodeAddress addr) const {
 223   return const_cast<CodeCache&>(*this).blockFor(addr);
 224 }
 225
 226 size_t CodeCache::totalUsed() const {
 227   size_t ret = 0;
 228   forEachBlock([&ret](const char*, const CodeBlock& b) {
 229     // A thread with the write lease may be modifying b.m_frontier while we
 230     // call b.used() but it should never modify b.m_base. This means that at
 231     // worst b.used() will return a slightly stale value.
 232     ret += b.used();
 233   });
 234   return ret;
 235 }
 236
 237 bool CodeCache::isValidCodeAddress(ConstCodeAddress addr) const {
 238   return addr >= m_base && addr < m_base + m_codeSize &&
 239     (addr < m_threadLocalStart ||
 240      addr >= m_threadLocalStart + m_threadLocalSize);
 241 }
 242
 243 void CodeCache::protect() {
 244   mprotect(m_base, m_codeSize, PROT_READ | PROT_EXEC);
 245 }
 246
 247 void CodeCache::unprotect() {
 248   mprotect(m_base, m_codeSize, PROT_READ | PROT_WRITE | PROT_EXEC);
 249 }
 250
 251 CodeCache::View CodeCache::view(TransKind kind) {
 252   auto view = [&] {
 253     if (isProfiling(kind)) {
 254       return View{m_prof, m_frozen, m_frozen, m_data, false};
 255     }
 256
 257     const bool isOpt = kind == TransKind::Optimize ||
 258                        kind == TransKind::OptPrologue;
 259     if (isOpt && m_useHot && m_hot.available() > kMinTranslationBytes) {
 260       return View{m_hot, m_cold, m_frozen, m_data, false};
 261     }
 262
 263     return View{m_main, m_cold, m_frozen, m_data, false};
 264   }();
 265
 266   tc::assertOwnsCodeLock(view);
 267   return view;
 268 }
 269
 270 }}