hphp/runtime/vm/jit/tc-region.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17
  18 #include "hphp/runtime/vm/jit/tc.h"
  19 #include "hphp/runtime/vm/jit/tc-internal.h"
  20 #include "hphp/runtime/vm/jit/tc-record.h"
  21 #include "hphp/runtime/vm/jit/tc-relocate.h"
  22
  23 #include "hphp/runtime/vm/jit/align.h"
  24 #include "hphp/runtime/vm/jit/cfg.h"
  25 #include "hphp/runtime/vm/jit/cg-meta.h"
  26 #include "hphp/runtime/vm/jit/code-cache.h"
  27 #include "hphp/runtime/vm/jit/func-guard.h"
  28 #include "hphp/runtime/vm/jit/func-prologue.h"
  29 #include "hphp/runtime/vm/jit/ir-unit.h"
  30 #include "hphp/runtime/vm/jit/mcgen.h"
  31 #include "hphp/runtime/vm/jit/prof-data.h"
  32 #include "hphp/runtime/vm/jit/relocation.h"
  33 #include "hphp/runtime/vm/jit/service-requests.h"
  34 #include "hphp/runtime/vm/jit/smashable-instr.h"
  35 #include "hphp/runtime/vm/jit/srcdb.h"
  36 #include "hphp/runtime/vm/jit/timer.h"
  37 #include "hphp/runtime/vm/jit/trans-db.h"
  38 #include "hphp/runtime/vm/jit/trans-rec.h"
  39 #include "hphp/runtime/vm/jit/vasm-emit.h"
  40 #include "hphp/runtime/vm/jit/vasm-gen.h"
  41 #include "hphp/runtime/vm/jit/vm-protect.h"
  42 #include "hphp/runtime/vm/jit/vtune-jit.h"
  43
  44 #include "hphp/util/service-data.h"
  45 #include "hphp/util/struct-log.h"
  46 #include "hphp/util/timer.h"
  47 #include "hphp/util/trace.h"
  48
  49 TRACE_SET_MOD(mcg);
  50
  51 namespace HPHP { namespace jit { namespace tc {
  52
  53 namespace {
  54
  55 /*
  56  * Attempt to emit code for the given IRUnit to `code'. Returns true on
  57  * success, false if codegen failed.
  58  */
  59 bool mcGenUnit(TransEnv& env, CodeCache::View codeView, CGMeta& fixups) {
  60   auto const& unit = *env.unit;
  61   try {
  62     emitVunit(*env.vunit, unit, codeView, fixups,
  63               mcgen::dumpTCAnnotation(*env.args.sk.func(), env.args.kind)
  64               ? &env.annotations
  65               : nullptr);
  66   } catch (const DataBlockFull& dbFull) {
  67     if (dbFull.name == "hot") {
  68       code().disableHot();
  69       return false;
  70     } else {
  71       always_assert_flog(0, "data block = {}\nmessage: {}\n",
  72                          dbFull.name, dbFull.what());
  73     }
  74   }
  75
  76   auto const startSk = unit.context().srcKey();
  77   if (unit.context().kind == TransKind::Profile) {
  78     profData()->setProfiling(startSk.func()->getFuncId());
  79   }
  80
  81   return true;
  82 }
  83
  84 TransLocMaker relocateLocalTranslation(TransRange range, TransKind kind,
  85                                        CodeCache::View srcView,
  86                                        CGMeta& fixups) {
  87   auto reloc = [&] () -> folly::Optional<TransLocMaker> {
  88     auto view = code().view(kind);
  89     TransLocMaker tlm(view);
  90     tlm.markStart();
  91
  92     try {
  93       RelocationInfo rel;
  94
  95       auto origin = range.data;
  96       if (!origin.empty()) {
  97         view.data().bytes(origin.size(),
  98                           srcView.data().toDestAddress(origin.begin()));
  99
 100         auto dest = tlm.dataRange();
 101         auto oAddr = origin.begin();
 102         auto dAddr = dest.begin();
 103         while (oAddr != origin.end()) {
 104           assertx(dAddr != dest.end());
 105           rel.recordAddress(oAddr++, dAddr++, 0);
 106         }
 107       }
 108
 109       relocate(rel, view.main(), range.main.begin(), range.main.end(),
 110                srcView.main(), fixups, nullptr);
 111       relocate(rel, view.cold(), range.cold.begin(), range.cold.end(),
 112                srcView.cold(), fixups, nullptr);
 113       if (&srcView.cold() != &srcView.frozen()) {
 114         relocate(rel, view.frozen(), range.frozen.begin(),
 115                  range.frozen.end(), srcView.frozen(), fixups, nullptr);
 116       }
 117
 118       adjustForRelocation(rel);
 119       adjustMetaDataForRelocation(rel, nullptr, fixups);
 120       adjustCodeForRelocation(rel, fixups);
 121     } catch (const DataBlockFull& dbFull) {
 122       if (dbFull.name == "hot") {
 123         code().disableHot();
 124         tlm.rollback();
 125         return folly::none;
 126       }
 127       always_assert_flog(0, "data block = {}\nmessage: {}\n",
 128                          dbFull.name, dbFull.what());
 129     }
 130     return tlm;
 131   };
 132
 133   if (auto tlm = reloc()) return *tlm;
 134
 135   auto tlm = reloc();
 136   always_assert(tlm);
 137   return *tlm;
 138 }
 139
 140 ////////////////////////////////////////////////////////////////////////////////
 141 }
 142
 143 SrcRec* findSrcRec(SrcKey sk) {
 144   return srcDB().find(sk);
 145 }
 146
 147 void createSrcRec(SrcKey sk, FPInvOffset spOff) {
 148   if (srcDB().find(sk)) return;
 149
 150   auto const srcRecSPOff = sk.resumed() ? folly::none
 151                                         : folly::make_optional(spOff);
 152
 153   // We put retranslate requests at the end of our slab to more frequently
 154   // allow conditional jump fall-throughs
 155   auto codeLock = lockCode();
 156   auto codeView = code().view();
 157   TCA astart = codeView.main().frontier();
 158   TCA coldStart = codeView.cold().frontier();
 159   TCA frozenStart = codeView.frozen().frontier();
 160   TCA req;
 161   if (!RuntimeOption::EvalEnableReusableTC) {
 162     req = svcreq::emit_persistent(codeView.cold(),
 163                                   codeView.data(),
 164                                   srcRecSPOff,
 165                                   REQ_RETRANSLATE,
 166                                   sk.offset(),
 167                                   TransFlags().packed);
 168   } else {
 169     auto const stubsize = svcreq::stub_size();
 170     auto newStart = codeView.cold().allocInner(stubsize);
 171     if (!newStart) {
 172       newStart = codeView.cold().frontier();
 173     }
 174     // Ensure that the anchor translation is a known size so that it can be
 175     // reclaimed when the function is freed
 176     req = svcreq::emit_ephemeral(codeView.cold(),
 177                                  codeView.data(),
 178                                  (TCA)newStart,
 179                                  srcRecSPOff,
 180                                  REQ_RETRANSLATE,
 181                                  sk.offset(),
 182                                  TransFlags().packed);
 183   }
 184   SKTRACE(1, sk, "inserting anchor translation for (%p,%d) at %p\n",
 185           sk.unit(), sk.offset(), req);
 186
 187   auto metaLock = lockMetadata();
 188   always_assert(srcDB().find(sk) == nullptr);
 189   auto const sr = srcDB().insert(sk);
 190   if (RuntimeOption::EvalEnableReusableTC) {
 191     recordFuncSrcRec(sk.func(), sr);
 192   }
 193   sr->setFuncInfo(sk.func());
 194   sr->setAnchorTranslation(req);
 195
 196   if (srcRecSPOff) always_assert(sr->nonResumedSPOff() == *srcRecSPOff);
 197
 198   size_t asize      = codeView.main().frontier()   - astart;
 199   size_t coldSize   = codeView.cold().frontier()   - coldStart;
 200   size_t frozenSize = codeView.frozen().frontier() - frozenStart;
 201   assertx(asize == 0);
 202   if (coldSize && RuntimeOption::EvalDumpTCAnchors) {
 203     auto const transID =
 204       profData() && transdb::enabled() ? profData()->allocTransID()
 205                                        : kInvalidTransID;
 206     TransRec tr(sk, transID, TransKind::Anchor,
 207                 astart, asize, coldStart, coldSize,
 208                 frozenStart, frozenSize);
 209     transdb::addTranslation(tr);
 210     if (RuntimeOption::EvalJitUseVtuneAPI) {
 211       reportTraceletToVtune(sk.unit(), sk.func(), tr);
 212     }
 213
 214     assertx(!transdb::enabled() ||
 215             transdb::getTransRec(coldStart)->kind == TransKind::Anchor);
 216   }
 217 }
 218
 219
 220 ////////////////////////////////////////////////////////////////////////////////
 221
 222 folly::Optional<TransMetaInfo> emitTranslation(TransEnv env, OptView optDst) {
 223   Timer timer(Timer::mcg_finishTranslation);
 224
 225   VMProtect _;
 226
 227   auto& args = env.args;
 228   auto const sk = args.sk;
 229
 230   profileSetHotFuncAttr();
 231
 232   std::unique_lock<SimpleMutex> codeLock;
 233   if (!optDst) {
 234     codeLock = lockCode();
 235   }
 236
 237   auto codeView = optDst ? *optDst : code().view(args.kind);
 238   auto viewKind = args.kind;
 239
 240   CGMeta fixups;
 241   TransLocMaker maker{codeView};
 242   maker.markStart();
 243
 244   // mcGenUnit emits machine code from vasm
 245   if (env.vunit && !mcGenUnit(env, codeView, fixups)) {
 246     // mcGenUnit() failed. Roll back, drop the unit and region, and clear
 247     // fixups.
 248     maker.rollback();
 249     maker.markStart();
 250     env.unit.reset();
 251     env.vunit.reset();
 252     args.region.reset();
 253     fixups.clear();
 254   }
 255
 256   if (env.vunit) {
 257     if (!newTranslation()) {
 258       return folly::none;
 259     }
 260   } else {
 261     args.kind = TransKind::Interp;
 262     FTRACE(1, "emitting dispatchBB interp request for failed "
 263            "translation (spOff = {})\n", env.initSpOffset.offset);
 264     vwrap(codeView.main(), codeView.data(), fixups,
 265           [&] (Vout& v) { emitInterpReq(v, sk, env.initSpOffset); },
 266           CodeKind::Helper);
 267   }
 268
 269   Timer metaTimer(Timer::mcg_finishTranslation_metadata);
 270   auto range = maker.markEnd();
 271
 272   if (args.kind == TransKind::Profile) {
 273     always_assert(args.region);
 274     auto metaLock = lockMetadata();
 275     profData()->addTransProfile(env.transID, args.region, env.pconds);
 276   }
 277
 278   TransRec tr;
 279   if (RuntimeOption::EvalJitUseVtuneAPI ||
 280       Trace::moduleEnabledRelease(Trace::trans, 1) ||
 281       transdb::enabled()) {
 282     tr = maker.rec(sk, env.transID, args.kind, args.region, fixups.bcMap,
 283                    std::move(env.annotations),
 284                    env.unit && cfgHasLoop(*env.unit));
 285   }
 286
 287   if (env.unit && env.unit->logEntry()) {
 288     auto metaLock = lockMetadata();
 289     logTranslation(env, range);
 290   }
 291
 292   return TransMetaInfo{sk, viewKind, args.kind, range, std::move(fixups),
 293                        std::move(tr)};
 294 }
 295
 296 folly::Optional<TransLoc>
 297 publishTranslation(TransMetaInfo info, OptView optSrcView) {
 298   auto const sk = info.sk;
 299   auto range = info.range;
 300   auto& fixups = info.meta;
 301   auto& tr = info.transRec;
 302   bool needsRelocate = RuntimeOption::EvalEnableOptTCBuffer &&
 303     !code().isValidCodeAddress(info.range.main.begin());
 304
 305   auto const srcRec = srcDB().find(sk);
 306   always_assert(srcRec);
 307
 308   auto checkLimit = [&] {
 309     auto const limit = info.viewKind == TransKind::Profile
 310       ? RuntimeOption::EvalJitMaxProfileTranslations
 311       : RuntimeOption::EvalJitMaxTranslations;
 312
 313     auto const numTrans = srcRec->translations().size();
 314
 315     // Once numTrans has reached limit + 1 we know that an interp translation
 316     // has already been emitted. Prior to that if numTrans == limit only allow
 317     // interp translations to avoid a race where multiple threads believe there
 318     // are still available translations.
 319     if (numTrans == limit + 1) return false;
 320     if (numTrans == limit && info.transKind != TransKind::Interp) return false;
 321     return true;
 322   };
 323
 324   if (!checkLimit()) return folly::none;
 325
 326   auto codeLock = lockCode();
 327   auto metaLock = lockMetadata();
 328
 329   // Check again now that we have the lock, the first check is racy but may
 330   // prevent unnecessarily acquiring the lock.
 331   if (!checkLimit()) return folly::none;
 332
 333   // 1) If we are currently in a thread local TC we need to relocate into the
 334   //    end of the real TC.
 335   // 2) If reusable TC is enabled we need to relocate to an inner allocation if
 336   //    sufficiently sized free blocks can be found
 337   //
 338   // Ideally we can combine (1) and (2), however, currently thread-local TC is
 339   // only useful in production environments where we may want to emit optimize
 340   // translations in parallel, while reusable TC is only meaningful in sandboxes
 341   // where translations may be invalidates by source modifications.
 342   auto view = [&] {
 343     if (!needsRelocate) return code().view(info.viewKind);
 344
 345     auto tlm = relocateLocalTranslation(range, info.viewKind, *optSrcView,
 346                                         fixups);
 347     range = tlm.markEnd();
 348     return tlm.view();
 349   }();
 350
 351   auto loc = range.loc();
 352   tryRelocateNewTranslation(sk, loc, view, fixups);
 353
 354   // Finally, record various metadata about the translation and add it to the
 355   // SrcRec.
 356   if (RuntimeOption::EvalProfileBC) {
 357     auto const vmUnit = sk.unit();
 358     TransBCMapping prev{};
 359     for (auto& cur : fixups.bcMap) {
 360       if (!cur.aStart) continue;
 361       if (prev.aStart) {
 362         if (prev.bcStart < vmUnit->bclen()) {
 363           recordBCInstr(uint32_t(vmUnit->getOp(prev.bcStart)),
 364                         prev.aStart, cur.aStart, false);
 365         }
 366       } else {
 367         recordBCInstr(OpTraceletGuard, loc.mainStart(), cur.aStart, false);
 368       }
 369       prev = cur;
 370     }
 371   }
 372
 373   recordRelocationMetaData(sk, *srcRec, loc, fixups);
 374   recordGdbTranslation(sk, sk.func(), view.main(), loc.mainStart(),
 375                        false, false);
 376   recordGdbTranslation(sk, sk.func(), view.cold(), loc.coldStart(),
 377                        false, false);
 378
 379   if (tr.isValid()) {
 380     tr.aStart = loc.mainStart();
 381     tr.acoldStart = loc.coldCodeStart();
 382     tr.afrozenStart = loc.frozenCodeStart();
 383     tr.aLen = loc.mainSize();
 384     tr.acoldLen = loc.coldSize();
 385     tr.afrozenLen = loc.frozenSize();
 386   }
 387
 388   transdb::addTranslation(tr);
 389   if (RuntimeOption::EvalJitUseVtuneAPI) {
 390     reportTraceletToVtune(sk.unit(), sk.func(), tr);
 391   }
 392
 393   GrowableVector<IncomingBranch> inProgressTailBranches;
 394   fixups.process(&inProgressTailBranches);
 395
 396   // SrcRec::newTranslation() makes this code reachable. Do this last;
 397   // otherwise there's some chance of hitting in the reader threads whose
 398   // metadata is not yet visible.
 399   TRACE(1, "newTranslation: %p  sk: %s\n",
 400         loc.mainStart(), showShort(sk).c_str());
 401   srcRec->newTranslation(loc, inProgressTailBranches);
 402
 403   TRACE(1, "mcg: %zd-byte tracelet\n", (ssize_t)loc.mainSize());
 404   if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
 405     Trace::traceRelease("%s", getTCSpace().c_str());
 406   }
 407
 408   reportJitMaturity(code());
 409
 410   return loc;
 411 }
 412
 413 void publishOptFunction(FuncMetaInfo info) {
 414   auto const func = info.func;
 415
 416   for (auto const rec : info.prologues) {
 417     emitFuncPrologueOpt(rec);
 418   }
 419
 420   mcgen::ReadThreadLocalTC localTC(info.tcBuf);
 421   for (auto& trans : info.translations) {
 422     auto const regionSk = trans.sk;
 423     auto const loc = publishTranslation(std::move(trans), info.tcBuf.view());
 424     if (loc &&
 425         regionSk.offset() == func->base() &&
 426         !func->hasThisVaries() &&
 427         func->getDVFunclets().size() == 0 &&
 428         func->getFuncBody() == ustubs().funcBodyHelperThunk) {
 429       func->setFuncBody(loc->mainStart());
 430     }
 431   }
 432 }
 433
 434 static void invalidateSrcKeyNoLock(SrcKey sk) {
 435   assertx(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
 436   /*
 437    * Reroute existing translations for SrcKey to an as-yet indeterminate
 438    * new one.
 439    */
 440   auto const sr = srcDB().find(sk);
 441   always_assert(sr);
 442   /*
 443    * Since previous translations aren't reachable from here, we know we
 444    * just created some garbage in the TC. We currently have no mechanism
 445    * to reclaim this.
 446    */
 447   FTRACE_MOD(Trace::reusetc, 1,
 448              "Replacing translations from sk: {} " "to SrcRec addr={}\n",
 449              showShort(sk), (void*)sr);
 450   Trace::Indent _i;
 451   sr->replaceOldTranslations();
 452 }
 453
 454 void invalidateSrcKey(SrcKey sk) {
 455   auto codeLock = lockCode();
 456   invalidateSrcKeyNoLock(sk);
 457 }
 458
 459 void invalidateFuncProfSrcKeys(const Func* func) {
 460   assertx(profData());
 461   auto const funcId = func->getFuncId();
 462   auto codeLock = lockCode();
 463   for (auto tid : profData()->funcProfTransIDs(funcId)) {
 464     invalidateSrcKeyNoLock(profData()->transRec(tid)->srcKey());
 465   }
 466 }
 467
 468 }}}