hphp/runtime/vm/jit/mc-generator.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/mc-generator.h"
  18 #include "hphp/runtime/vm/jit/vtune-jit.h"
  19
  20 #include <cinttypes>
  21 #include <assert.h>
  22 #include <stdarg.h>
  23 #include <stdint.h>
  24 #include <stdio.h>
  25 #ifndef _MSC_VER
  26 #include <unwind.h>
  27 #endif
  28
  29 #include <algorithm>
  30 #include <exception>
  31 #include <memory>
  32 #include <queue>
  33 #include <string>
  34 #include <strstream>
  35 #include <unordered_set>
  36 #include <vector>
  37
  38 #include <folly/Format.h>
  39 #include <folly/MapUtil.h>
  40 #include <folly/Optional.h>
  41 #include <folly/String.h>
  42 #include <folly/portability/SysMman.h>
  43 #include <folly/portability/Unistd.h>
  44
  45 #include "hphp/util/abi-cxx.h"
  46 #include "hphp/util/asm-x64.h"
  47 #include "hphp/util/bitops.h"
  48 #include "hphp/util/cycles.h"
  49 #include "hphp/util/debug.h"
  50 #include "hphp/util/disasm.h"
  51 #include "hphp/util/eh-frame.h"
  52 #include "hphp/util/logger.h"
  53 #include "hphp/util/maphuge.h"
  54 #include "hphp/util/meta.h"
  55 #include "hphp/util/process.h"
  56 #include "hphp/util/rank.h"
  57 #include "hphp/util/build-info.h"
  58 #include "hphp/util/ringbuffer.h"
  59 #include "hphp/util/safe-cast.h"
  60 #include "hphp/util/service-data.h"
  61 #include "hphp/util/struct-log.h"
  62 #include "hphp/util/timer.h"
  63 #include "hphp/util/trace.h"
  64
  65 #include "hphp/vixl/a64/constants-a64.h"
  66 #include "hphp/vixl/a64/macro-assembler-a64.h"
  67
  68 #include "hphp/runtime/base/arch.h"
  69 #include "hphp/runtime/base/execution-context.h"
  70 #include "hphp/runtime/base/rds.h"
  71 #include "hphp/runtime/base/runtime-option.h"
  72 #include "hphp/runtime/base/stats.h"
  73 #include "hphp/runtime/base/strings.h"
  74 #include "hphp/runtime/base/zend-string.h"
  75 #include "hphp/runtime/vm/blob-helper.h"
  76 #include "hphp/runtime/vm/bytecode.h"
  77 #include "hphp/runtime/vm/debug/debug.h"
  78 #include "hphp/runtime/vm/func.h"
  79 #include "hphp/runtime/vm/hhbc-codec.h"
  80 #include "hphp/runtime/vm/member-operations.h"
  81 #include "hphp/runtime/vm/php-debug.h"
  82 #include "hphp/runtime/vm/repo.h"
  83 #include "hphp/runtime/vm/runtime.h"
  84 #include "hphp/runtime/vm/srckey.h"
  85 #include "hphp/runtime/vm/treadmill.h"
  86 #include "hphp/runtime/vm/type-profile.h"
  87 #include "hphp/runtime/vm/unwind.h"
  88
  89 #include "hphp/runtime/vm/jit/align.h"
  90 #include "hphp/runtime/vm/jit/cg-meta.h"
  91 #include "hphp/runtime/vm/jit/check.h"
  92 #include "hphp/runtime/vm/jit/code-cache.h"
  93 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  94 #include "hphp/runtime/vm/jit/debug-guards.h"
  95 #include "hphp/runtime/vm/jit/func-guard.h"
  96 #include "hphp/runtime/vm/jit/func-prologue.h"
  97 #include "hphp/runtime/vm/jit/inlining-decider.h"
  98 #include "hphp/runtime/vm/jit/irgen.h"
  99 #include "hphp/runtime/vm/jit/irlower.h"
 100 #include "hphp/runtime/vm/jit/location.h"
 101 #include "hphp/runtime/vm/jit/normalized-instruction.h"
 102 #include "hphp/runtime/vm/jit/perf-counters.h"
 103 #include "hphp/runtime/vm/jit/print.h"
 104 #include "hphp/runtime/vm/jit/prof-data.h"
 105 #include "hphp/runtime/vm/jit/punt.h"
 106 #include "hphp/runtime/vm/jit/recycle-tc.h"
 107 #include "hphp/runtime/vm/jit/region-selection.h"
 108 #include "hphp/runtime/vm/jit/relocation.h"
 109 #include "hphp/runtime/vm/jit/service-requests.h"
 110 #include "hphp/runtime/vm/jit/smashable-instr.h"
 111 #include "hphp/runtime/vm/jit/srcdb.h"
 112 #include "hphp/runtime/vm/jit/tc-info.h"
 113 #include "hphp/runtime/vm/jit/timer.h"
 114 #include "hphp/runtime/vm/jit/translate-region.h"
 115 #include "hphp/runtime/vm/jit/translator-inline.h"
 116 #include "hphp/runtime/vm/jit/unwind-itanium.h"
 117 #include "hphp/runtime/vm/jit/vasm-emit.h"
 118 #include "hphp/runtime/vm/jit/vasm-gen.h"
 119 #include "hphp/runtime/vm/jit/vasm-instr.h"
 120
 121 #include "hphp/runtime/ext/generator/ext_generator.h"
 122 #include "hphp/runtime/ext/std/ext_std_function.h"
 123 #include "hphp/runtime/server/http-server.h"
 124 #include "hphp/runtime/server/source-root-info.h"
 125
 126 namespace HPHP { namespace jit {
 127
 128 ///////////////////////////////////////////////////////////////////////////////
 129
 130 TRACE_SET_MOD(mcg);
 131
 132 using namespace Trace;
 133
 134 // The global MCGenerator object.
 135 MCGenerator* mcg;
 136
 137 static __thread size_t s_initialTCSize;
 138 static ServiceData::ExportedCounter* s_jitMaturityCounter;
 139 static std::atomic<bool> s_loggedJitMature{false};
 140
 141 ///////////////////////////////////////////////////////////////////////////////
 142
 143 namespace {
 144 /*
 145  * Convenience class for creating TransLocs and TransRecs for new translations.
 146  *
 147  * Records the beginning and end of a translation and stores the size of the
 148  * cold and frozen regions in the first 4 bytes of their respective regions.
 149  */
 150 struct TransLocMaker {
 151   explicit TransLocMaker(CodeCache::View c) : cache(c) {}
 152
 153   /*
 154    * Record the start of a translation, and reserve space at the top of cold
 155    * and frozen (if they aren't the same) to record sizes.
 156    */
 157   void markStart() {
 158     loc.setMainStart(cache.main().frontier());
 159     loc.setColdStart(cache.cold().frontier());
 160     loc.setFrozenStart(cache.frozen().frontier());
 161     dataStart = cache.data().frontier();
 162
 163     cache.cold().dword(0);
 164     if (&cache.cold() != &cache.frozen()) cache.frozen().dword(0);
 165   }
 166
 167   /*
 168    * If loc contains a valid location, reset the frontiers of all code and data
 169    * blocks to the positions recorded by the last call to markStart().
 170    */
 171   void rollback() {
 172     if (loc.empty()) return;
 173
 174     cache.main().setFrontier(loc.mainStart());
 175     cache.cold().setFrontier(loc.coldStart());
 176     cache.frozen().setFrontier(loc.frozenStart());
 177     cache.data().setFrontier(dataStart);
 178   }
 179
 180   /*
 181    * Record the end of a translation, storing the size of cold and frozen,
 182    * returns a TransLoc representing the translation.
 183    */
 184   TransLoc markEnd() {
 185     uint32_t* coldSize   = (uint32_t*)loc.coldStart();
 186     uint32_t* frozenSize = (uint32_t*)loc.frozenStart();
 187     *coldSize   = cache  .cold().frontier() - loc.coldStart();
 188     *frozenSize = cache.frozen().frontier() - loc.frozenStart();
 189     loc.setMainSize(cache.main().frontier() - loc.mainStart());
 190
 191     return loc;
 192   }
 193
 194   /*
 195    * Create a TransRec for the translation, markEnd() should be called prior to
 196    * calling rec().
 197    */
 198   TransRec rec(
 199       SrcKey                      sk,
 200       TransID                     transID,
 201       TransKind                   kind,
 202       RegionDescPtr               region  = RegionDescPtr(),
 203       std::vector<TransBCMapping> bcmap   = std::vector<TransBCMapping>(),
 204       Annotations&&               annot   = Annotations(),
 205       bool                        hasLoop = false) const {
 206     return TransRec(sk, transID, kind,
 207                     loc.mainStart(), loc.mainSize(),
 208                     loc.coldCodeStart(), loc.coldCodeSize(),
 209                     loc.frozenCodeStart(), loc.frozenCodeSize(),
 210                     std::move(region), std::move(bcmap),
 211                     std::move(annot), hasLoop);
 212   }
 213
 214 private:
 215   CodeCache::View cache;
 216   TransLoc loc;
 217   Address dataStart;
 218 };
 219 }
 220
 221 ///////////////////////////////////////////////////////////////////////////////
 222
 223 bool shouldPGOFunc(const Func& func) {
 224   if (profData() == nullptr) return false;
 225
 226   // JITing pseudo-mains requires extra checks that blow the IR.  PGO
 227   // can significantly increase the size of the regions, so disable it for
 228   // pseudo-mains (so regions will be just tracelets).
 229   if (func.isPseudoMain()) return false;
 230
 231   if (!RuntimeOption::EvalJitPGOHotOnly) return true;
 232   return func.attrs() & AttrHot;
 233 }
 234
 235 bool dumpTCAnnotation(const Func& func, TransKind transKind) {
 236   return RuntimeOption::EvalDumpTCAnnotationsForAllTrans ||
 237     (transKind == TransKind::Optimize && (func.attrs() & AttrHot));
 238 }
 239
 240 ///////////////////////////////////////////////////////////////////////////////
 241
 242 bool MCGenerator::profileSrcKey(SrcKey sk) const {
 243   if (!shouldPGOFunc(*sk.func())) return false;
 244   if (profData()->optimized(sk.funcID())) return false;
 245   if (profData()->profiling(sk.funcID())) return true;
 246
 247   // Don't start profiling new functions if the size of either main or
 248   // prof is already above Eval.JitAMaxUsage and we already filled hot.
 249   auto tcUsage = std::max(m_code.main().used(), m_code.prof().used());
 250   if (tcUsage >= CodeCache::AMaxUsage && !m_code.hotEnabled()) {
 251     return false;
 252   }
 253
 254   return requestCount() <= RuntimeOption::EvalJitProfileRequests;
 255 }
 256
 257 /*
 258  * Invalidate the SrcDB entries for func's SrcKeys that have any
 259  * Profile translation.
 260  */
 261 void MCGenerator::invalidateFuncProfSrcKeys(const Func* func) {
 262   assertx(profData());
 263   FuncId funcId = func->getFuncId();
 264   for (auto tid : profData()->funcProfTransIDs(funcId)) {
 265     invalidateSrcKey(profData()->transRec(tid)->srcKey());
 266   }
 267 }
 268
 269 TransResult MCGenerator::retranslate(const TransArgs& args) {
 270   auto sr = m_srcDB.find(args.sk);
 271   always_assert(sr);
 272   bool locked = sr->tryLock();
 273   SCOPE_EXIT {
 274     if (locked) sr->freeLock();
 275   };
 276   if (isDebuggerAttachedProcess() && m_tx.isSrcKeyInBL(args.sk)) {
 277     // We are about to translate something known to be blacklisted by
 278     // debugger, exit early
 279     SKTRACE(1, args.sk, "retranslate abort due to debugger\n");
 280     return nullptr;
 281   }
 282
 283   LeaseHolder writer(Translator::WriteLease(), args.sk.func());
 284   if (!writer.canTranslate() ||
 285       !shouldTranslate(args.sk.func(), args.kind)) {
 286     return nullptr;
 287   }
 288   if (!locked) {
 289     // Even though we knew above that we were going to skip doing another
 290     // translation, we wait until we get the write lease, to avoid spinning
 291     // through the tracelet guards again and again while another thread is
 292     // writing to it.
 293     return sr->getTopTranslation();
 294   }
 295   if (sr->translations().size() > RuntimeOption::EvalJitMaxTranslations) {
 296     always_assert(sr->translations().size() ==
 297                   RuntimeOption::EvalJitMaxTranslations + 1);
 298     return sr->getTopTranslation();
 299   }
 300   SKTRACE(1, args.sk, "retranslate\n");
 301
 302   auto newArgs = args;
 303   newArgs.kind = profileSrcKey(args.sk) ? TransKind::Profile : TransKind::Live;
 304
 305   // We only allow concurrent jitting of non-Profile translations if
 306   // EvalJitConcurrently is >= 2.
 307   if (RuntimeOption::EvalJitConcurrently < 2 &&
 308       newArgs.kind != TransKind::Profile &&
 309       !writer.acquire()) {
 310     return nullptr;
 311   }
 312
 313   auto result = translate(newArgs);
 314
 315   checkFreeProfData();
 316   return result;
 317 }
 318
 319 TCA MCGenerator::retranslateOpt(TransID transId, bool align) {
 320   if (profData() == nullptr) return nullptr;
 321   if (isDebuggerAttachedProcess()) return nullptr;
 322
 323   // Until the rest of this function is cleaned up enough to support concurrent
 324   // retranslation of different functions, we grab both the global write lease
 325   // and a Func-specific lease for the current Func.
 326   LeaseHolder writer(Translator::WriteLease());
 327   if (!writer.canWrite()) return nullptr;
 328
 329   auto rec = profData()->transRec(transId);
 330   always_assert(rec);
 331   always_assert(rec->region() != nullptr);
 332   LeaseHolder funcWriter(Translator::WriteLease(), rec->func());
 333   if (!funcWriter.canTranslate()) return nullptr;
 334
 335   TRACE(1, "retranslateOpt: transId = %u\n", transId);
 336
 337   auto func   = rec->func();
 338   auto funcId = func->getFuncId();
 339   auto sk     = rec->srcKey();
 340
 341   if (profData()->optimized(funcId)) return nullptr;
 342   profData()->setOptimized(funcId);
 343
 344   func->setFuncBody(ustubs().funcBodyHelperThunk);
 345
 346   // Invalidate SrcDB's entries for all func's SrcKeys.
 347   invalidateFuncProfSrcKeys(func);
 348
 349   // Regenerate the prologues and DV funclets before the actual function body.
 350   bool includedBody{false};
 351   TCA start = regeneratePrologues(func, sk, includedBody);
 352
 353   // Regionize func and translate all its regions.
 354   std::string transCFGAnnot;
 355   auto const regions = includedBody ? std::vector<RegionDescPtr>{}
 356                                     : regionizeFunc(func, this, transCFGAnnot);
 357
 358   for (auto region : regions) {
 359     always_assert(!region->empty());
 360     auto regionSk = region->start();
 361     auto transArgs = TransArgs{regionSk};
 362     if (transCFGAnnot.size() > 0) {
 363       transArgs.annotations.emplace_back("TransCFG", transCFGAnnot);
 364     }
 365     transArgs.region = region;
 366     transArgs.kind = TransKind::Optimize;
 367
 368     auto const regionStart = translate(transArgs).tca();
 369     if (regionStart != nullptr &&
 370         regionSk.offset() == func->base() &&
 371         func->getDVFunclets().size() == 0 &&
 372         func->getFuncBody() == ustubs().funcBodyHelperThunk) {
 373       func->setFuncBody(regionStart);
 374     }
 375     if (start == nullptr && regionSk == sk) {
 376       start = regionStart;
 377     }
 378     transCFGAnnot = ""; // so we don't annotate it again
 379   }
 380
 381   checkFreeProfData();
 382   return start;
 383 }
 384
 385 void MCGenerator::checkFreeProfData() {
 386   // In PGO mode, we free all the profiling data once the main area code reaches
 387   // its maximum usage and either the hot area is also full or all the functions
 388   // that were profiled have already been optimized.
 389   if (profData() &&
 390       !RuntimeOption::EvalEnableReusableTC &&
 391       m_code.main().used() >= CodeCache::AMaxUsage &&
 392       (!m_code.hotEnabled() || profData()->optimizedAllProfiledFuncs())) {
 393     discardProfData();
 394   }
 395 }
 396
 397 static bool liveFrameIsPseudoMain() {
 398   auto const ar = vmfp();
 399   if (!(ar->func()->attrs() & AttrMayUseVV)) return false;
 400   return ar->hasVarEnv() && ar->getVarEnv()->isGlobalScope();
 401 }
 402
 403 /*
 404  * Return an existing translation for `args', or nullptr if one can't be found.
 405  */
 406 TCA MCGenerator::findTranslation(const TransArgs& args) const {
 407   auto sk = args.sk;
 408   sk.func()->validate();
 409
 410   if (liveFrameIsPseudoMain() && !RuntimeOption::EvalJitPseudomain) {
 411     SKTRACE(2, sk, "punting on pseudoMain\n");
 412     return nullptr;
 413   }
 414
 415   if (auto const sr = m_srcDB.find(sk)) {
 416     if (auto const tca = sr->getTopTranslation()) {
 417       SKTRACE(2, sk, "getTranslation: found %p\n", tca);
 418       return tca;
 419     }
 420   }
 421
 422   return nullptr;
 423 }
 424
 425 /*
 426  * Find or create a translation for `args'. Returns TCA of "best" current
 427  * translation. May return nullptr if it is currently impossible to create a
 428  * translation.
 429  */
 430 TransResult MCGenerator::getTranslation(const TransArgs& args) {
 431   if (auto const tca = findTranslation(args)) return tca;
 432
 433   return createTranslation(args);
 434 }
 435
 436 int MCGenerator::numTranslations(SrcKey sk) const {
 437   if (const SrcRec* sr = m_srcDB.find(sk)) {
 438     return sr->translations().size();
 439   }
 440   return 0;
 441 }
 442
 443 const StaticString
 444   s_php_errormsg("php_errormsg"),
 445   s_http_response_header("http_response_header");
 446
 447 bool MCGenerator::shouldTranslateNoSizeLimit(const Func* func) const {
 448   // If we've hit Eval.JitGlobalTranslationLimit, then we stop translating.
 449   if (m_numTrans >= RuntimeOption::EvalJitGlobalTranslationLimit) {
 450     return false;
 451   }
 452
 453   // Do not translate functions from units marked as interpret-only.
 454   if (func->unit()->isInterpretOnly()) {
 455     return false;
 456   }
 457
 458   /*
 459    * We don't support JIT compiling functions that use some super-dynamic php
 460    * variables.
 461    */
 462   if (func->lookupVarId(s_php_errormsg.get()) != -1 ||
 463       func->lookupVarId(s_http_response_header.get()) != -1) {
 464     return false;
 465   }
 466
 467   return true;
 468 }
 469
 470 bool MCGenerator::shouldTranslate(const Func* func, TransKind kind) const {
 471   if (!shouldTranslateNoSizeLimit(func)) return false;
 472
 473   // Otherwise, follow the Eval.JitAMaxUsage limit.  However, we do allow PGO
 474   // translations past that limit if there's still space in code.hot.
 475   if (m_code.main().used() < CodeCache::AMaxUsage) return true;
 476
 477   switch (kind) {
 478     case TransKind::ProfPrologue:
 479     case TransKind::Profile:
 480     case TransKind::OptPrologue:
 481     case TransKind::Optimize:
 482       return m_code.hotEnabled();
 483
 484     default:
 485       return false;
 486   }
 487 }
 488
 489
 490 static void populateLiveContext(RegionContext& ctx) {
 491   auto const fp = vmfp();
 492   auto const sp = vmsp();
 493
 494   always_assert(ctx.func == fp->m_func);
 495
 496   // Track local types.
 497   for (uint32_t i = 0; i < fp->m_func->numLocals(); ++i) {
 498     ctx.liveTypes.push_back(
 499       { Location::Local{i}, typeFromTV(frame_local(fp, i)) }
 500     );
 501     FTRACE(2, "added live type {}\n", show(ctx.liveTypes.back()));
 502   }
 503
 504   // Track stack types and pre-live ActRecs.
 505   int32_t stackOff = 0;
 506   visitStackElems(
 507     fp, sp, ctx.bcOffset,
 508     [&] (const ActRec* ar, Offset) {
 509       auto const objOrCls =
 510         ar->hasThis()  ? Type::SubObj(ar->getThis()->getVMClass()) :
 511         ar->hasClass() ? Type::SubCls(ar->getClass())
 512                        : TNullptr;
 513
 514       ctx.preLiveARs.push_back({ stackOff, ar->m_func, objOrCls });
 515       FTRACE(2, "added prelive ActRec {}\n", show(ctx.preLiveARs.back()));
 516       stackOff += kNumActRecCells;
 517     },
 518     [&] (const TypedValue* tv) {
 519       ctx.liveTypes.push_back(
 520         { Location::Stack{ctx.spOffset - stackOff}, typeFromTV(tv) }
 521       );
 522       stackOff++;
 523       FTRACE(2, "added live type {}\n", show(ctx.liveTypes.back()));
 524     }
 525   );
 526 }
 527
 528 TransResult MCGenerator::createTranslation(const TransArgs& args) {
 529   if (!shouldTranslate(args.sk.func(), args.kind)) return nullptr;
 530
 531   /*
 532    * Try to become the writer. We delay this until we *know* we will have a
 533    * need to create new translations, instead of just trying to win the lottery
 534    * at the dawn of time. Hopefully lots of requests won't require any new
 535    * translation.
 536    */
 537   auto sk = args.sk;
 538   LeaseHolder writer(Translator::WriteLease(), sk.func());
 539   if (!writer.canTranslate() ||
 540       !shouldTranslate(args.sk.func(), args.kind)) {
 541     return nullptr;
 542   }
 543
 544   if (RuntimeOption::EvalFailJitPrologs && sk.op() == Op::FCallAwait) {
 545     return nullptr;
 546   }
 547
 548   if (!createSrcRec(sk)) return nullptr;
 549
 550   auto sr = m_srcDB.find(sk);
 551   always_assert(sr);
 552
 553   if (auto const tca = sr->getTopTranslation()) {
 554     // Handle extremely unlikely race; someone may have just added the first
 555     // translation for this SrcRec while we did a non-blocking wait on the
 556     // write lease in createSrcRec().
 557     return tca;
 558   }
 559
 560   return retranslate(args);
 561 }
 562
 563 bool MCGenerator::createSrcRec(SrcKey sk) {
 564   if (m_srcDB.find(sk)) return true;
 565
 566   auto const srcRecSPOff = sk.resumed() ? folly::none
 567                                         : folly::make_optional(liveSpOff());
 568
 569   // We put retranslate requests at the end of our slab to more frequently
 570   // allow conditional jump fall-throughs
 571   auto code       = m_code.view();
 572   TCA astart      = code.main().frontier();
 573   TCA coldStart   = code.cold().frontier();
 574   TCA frozenStart = code.frozen().frontier();
 575   TCA req;
 576   if (!RuntimeOption::EvalEnableReusableTC) {
 577     req = svcreq::emit_persistent(code.cold(),
 578                                   code.data(),
 579                                   srcRecSPOff,
 580                                   REQ_RETRANSLATE,
 581                                   sk.offset(),
 582                                   TransFlags().packed);
 583   } else {
 584     auto const stubsize = svcreq::stub_size();
 585     auto newStart = code.cold().allocInner(stubsize);
 586     if (!newStart) {
 587       newStart = code.cold().frontier();
 588     }
 589     // Ensure that the anchor translation is a known size so that it can be
 590     // reclaimed when the function is freed
 591     req = svcreq::emit_ephemeral(code.cold(),
 592                                  code.data(),
 593                                  (TCA)newStart,
 594                                  srcRecSPOff,
 595                                  REQ_RETRANSLATE,
 596                                  sk.offset(),
 597                                  TransFlags().packed);
 598   }
 599   SKTRACE(1, sk, "inserting anchor translation for (%p,%d) at %p\n",
 600           sk.unit(), sk.offset(), req);
 601
 602   always_assert(m_srcDB.find(sk) == nullptr);
 603   auto const sr = m_srcDB.insert(sk);
 604   if (RuntimeOption::EvalEnableReusableTC) {
 605     recordFuncSrcRec(sk.func(), sr);
 606   }
 607   sr->setFuncInfo(sk.func());
 608   sr->setAnchorTranslation(req);
 609
 610   if (srcRecSPOff) always_assert(sr->nonResumedSPOff() == *srcRecSPOff);
 611
 612   size_t asize      = code.main().frontier()   - astart;
 613   size_t coldSize   = code.cold().frontier()   - coldStart;
 614   size_t frozenSize = code.frozen().frontier() - frozenStart;
 615   assertx(asize == 0);
 616   if (coldSize && RuntimeOption::EvalDumpTCAnchors) {
 617     auto const transID =
 618       profData() && Translator::isTransDBEnabled() ? profData()->allocTransID()
 619                                                    : kInvalidTransID;
 620     TransRec tr(sk, transID, TransKind::Anchor,
 621                 astart, asize, coldStart, coldSize,
 622                 frozenStart, frozenSize);
 623     m_tx.addTranslation(tr);
 624     if (RuntimeOption::EvalJitUseVtuneAPI) {
 625       reportTraceletToVtune(sk.unit(), sk.func(), tr);
 626     }
 627
 628     assertx(!m_tx.isTransDBEnabled() ||
 629            m_tx.getTransRec(coldStart)->kind == TransKind::Anchor);
 630   }
 631
 632   return true;
 633 }
 634
 635 TCA
 636 MCGenerator::lookupTranslation(SrcKey sk) const {
 637   if (auto const sr = m_srcDB.find(sk)) {
 638     return sr->getTopTranslation();
 639   }
 640   return nullptr;
 641 }
 642
 643 namespace {
 644 /*
 645  * Returns true iff we already have Eval.JitMaxTranslations translations
 646  * recorded in srcRec.
 647  */
 648 bool reachedTranslationLimit(SrcKey sk, const SrcRec& srcRec) {
 649   if (srcRec.translations().size() != RuntimeOption::EvalJitMaxTranslations) {
 650     return false;
 651   }
 652   INC_TPC(max_trans);
 653
 654   if (debug && Trace::moduleEnabled(Trace::mcg, 2)) {
 655     const auto& tns = srcRec.translations();
 656     TRACE(1, "Too many (%zd) translations: %s, BC offset %d\n",
 657           tns.size(), sk.unit()->filepath()->data(),
 658           sk.offset());
 659     SKTRACE(2, sk, "{\n");
 660     TCA topTrans = srcRec.getTopTranslation();
 661     for (size_t i = 0; i < tns.size(); ++i) {
 662       auto const rec = mcg->tx().getTransRec(tns[i].mainStart());
 663       assertx(rec);
 664       SKTRACE(2, sk, "%zd %p\n", i, tns[i].mainStart());
 665       if (tns[i].mainStart() == topTrans) {
 666         SKTRACE(2, sk, "%zd: *Top*\n", i);
 667       }
 668       if (rec->kind == TransKind::Anchor) {
 669         SKTRACE(2, sk, "%zd: Anchor\n", i);
 670       } else {
 671         SKTRACE(2, sk, "%zd: guards {\n", i);
 672         for (unsigned j = 0; j < rec->guards.size(); ++j) {
 673           FTRACE(2, "{}\n", rec->guards[j]);
 674         }
 675         SKTRACE(2, sk, "%zd } guards\n", i);
 676       }
 677     }
 678     SKTRACE(2, sk, "} /* Too many translations */\n");
 679   }
 680   return true;
 681 }
 682
 683 /*
 684  * Analyze the given TransArgs and return the region to translate, or nullptr
 685  * if one could not be selected.
 686  */
 687 RegionDescPtr prepareRegion(const TransArgs& args) {
 688   if (args.kind == TransKind::Optimize) {
 689     assertx(RuntimeOption::EvalJitPGO);
 690     if (args.region) return args.region;
 691
 692     assertx(isValidTransID(args.transId));
 693     return selectHotRegion(args.transId, mcg);
 694   }
 695
 696   // Attempt to create a region at this SrcKey
 697   assertx(args.kind == TransKind::Profile || args.kind == TransKind::Live);
 698   auto const sk = args.sk;
 699   RegionContext rContext { sk.func(), sk.offset(), liveSpOff(),
 700                            sk.resumed() };
 701   FTRACE(2, "populating live context for region\n");
 702   populateLiveContext(rContext);
 703   return selectRegion(rContext, args.kind);
 704 }
 705 }
 706
 707 TransResult MCGenerator::translate(TransArgs args) {
 708   INC_TPC(translate);
 709
 710   assert(args.kind != TransKind::Invalid);
 711   assertx(((uintptr_t)vmsp() & (sizeof(Cell) - 1)) == 0);
 712   assertx(((uintptr_t)vmfp() & (sizeof(Cell) - 1)) == 0);
 713
 714   if (!shouldTranslate(args.sk.func(), args.kind)) return nullptr;
 715
 716   Timer timer(Timer::mcg_translate);
 717
 718   auto const srcRec = m_srcDB.find(args.sk);
 719   always_assert(srcRec);
 720
 721   args.region = reachedTranslationLimit(args.sk, *srcRec) ? nullptr
 722                                                           : prepareRegion(args);
 723   TransEnv env{args};
 724   env.initSpOffset = args.region ? args.region->entry()->initialSpOffset()
 725                                  : liveSpOff();
 726   env.annotations.insert(env.annotations.end(),
 727                          args.annotations.begin(), args.annotations.end());
 728
 729   // Lower the RegionDesc to an IRUnit, then lower that to a Vunit.
 730   if (args.region) {
 731     if (args.kind == TransKind::Profile ||
 732         (profData() && Translator::isTransDBEnabled())) {
 733       env.transID = profData()->allocTransID();
 734     }
 735     auto const transContext =
 736       TransContext{env.transID, args.kind, args.flags, args.sk,
 737                    env.initSpOffset};
 738
 739     env.unit = irGenRegion(*args.region, transContext,
 740                            env.pconds, env.annotations);
 741     if (env.unit) {
 742       env.vunit = irlower::lowerUnit(*env.unit);
 743     }
 744   }
 745
 746   // If our caller requested partial translation and we don't already have the
 747   // write lease, we're done here. Otherwise, finish the translation now.
 748   if (args.allowPartial && !Translator::WriteLease().amOwner()) {
 749     return std::move(env);
 750   }
 751
 752   timer.stop();
 753   return finishTranslation(std::move(env));
 754 }
 755
 756 TCA MCGenerator::getFuncBody(Func* func) {
 757   TCA tca = func->getFuncBody();
 758   if (tca != ustubs().funcBodyHelperThunk) return tca;
 759
 760   DVFuncletsVec dvs = func->getDVFunclets();
 761
 762   if (dvs.size()) {
 763     LeaseHolder writer(Translator::WriteLease());
 764     if (!writer.canWrite()) return nullptr;
 765     tca = func->getFuncBody();
 766     if (tca != ustubs().funcBodyHelperThunk) return tca;
 767     tca = genFuncBodyDispatch(func, dvs, m_code.view());
 768     func->setFuncBody(tca);
 769   } else {
 770     SrcKey sk(func, func->base(), false);
 771     tca = mcg->getTranslation(TransArgs{sk}).tca();
 772     if (tca) func->setFuncBody(tca);
 773   }
 774
 775   return tca;
 776 }
 777
 778 /*
 779  * funcPrologue --
 780  *
 781  * Given a callee and a number of args, match up to the callee's
 782  * argument expectations and dispatch.
 783  *
 784  * Call/return hand-shaking is a bit funny initially. At translation time,
 785  * we don't necessarily know what function we're calling. For instance,
 786  *
 787  *   f(g());
 788  *
 789  * Will lead to a set of basic blocks like:
 790  *
 791  * b1: pushfuncd "f"
 792  *     pushfuncd "g"
 793  *     fcall
 794  * b2: fcall
 795  *
 796  * The fcall labelled "b2" above may not be statically bindable in our
 797  * execution model.
 798  *
 799  * We decouple the call work into a per-callsite portion, responsible
 800  * for recording the return address, and a per-(callee, numArgs) portion,
 801  * responsible for fixing up arguments and dispatching to remaining
 802  * code. We call the per-callee portion a "prologue."
 803  *
 804  * Also, we are called from two distinct environments. From REQ_BIND_CALL,
 805  * we're running "between" basic blocks, with all VM registers sync'ed.
 806  * However, we're also called in the middle of basic blocks, when dropping
 807  * entries into func->m_prologues. So don't go around using the
 808  * translation-time values of vmfp()/vmsp(), since they have an
 809  * unpredictable relationship to the source.
 810  */
 811 bool
 812 MCGenerator::checkCachedPrologue(const Func* func, int paramIdx,
 813                                  TCA& prologue) const {
 814   prologue = (TCA)func->getPrologue(paramIdx);
 815   if (prologue != ustubs().fcallHelperThunk) {
 816     TRACE(1, "cached prologue %s(%d) -> cached %p\n",
 817           func->fullName()->data(), paramIdx, prologue);
 818     assertx(m_code.isValidCodeAddress(prologue));
 819     return true;
 820   }
 821   return false;
 822 }
 823
 824 TCA MCGenerator::emitFuncPrologue(Func* func, int argc,
 825                                   bool forRegeneratePrologue) {
 826   const int nparams = func->numNonVariadicParams();
 827   const int paramIndex = argc <= nparams ? argc : nparams + 1;
 828
 829   auto const funcBody = SrcKey{func, func->getEntryForNumArgs(argc), false};
 830   auto const kind = profileSrcKey(funcBody) ? TransKind::ProfPrologue :
 831                     forRegeneratePrologue   ? TransKind::OptPrologue  :
 832                                               TransKind::LivePrologue;
 833
 834   profileSetHotFuncAttr();
 835   auto code = m_code.view(kind);
 836   TCA mainOrig = code.main().frontier();
 837   CGMeta fixups;
 838
 839   // If we're close to a cache line boundary, just burn some space to
 840   // try to keep the func and its body on fewer total lines.
 841   align(code.main(), &fixups, Alignment::CacheLineRoundUp, AlignContext::Dead);
 842
 843   TransLocMaker maker(code);
 844   maker.markStart();
 845
 846   // Careful: this isn't necessarily the real entry point. For funcIsMagic
 847   // prologues, this is just a possible prologue.
 848   TCA aStart = code.main().frontier();
 849
 850   // Give the prologue a TransID if we have profiling data.
 851   auto const transID = [&]{
 852     if (kind == TransKind::ProfPrologue) {
 853       auto const profData = jit::profData();
 854       auto const id = profData->allocTransID();
 855       profData->addTransProfPrologue(id, funcBody, paramIndex);
 856       return id;
 857     }
 858     if (profData() && Translator::isTransDBEnabled()) {
 859       return profData()->allocTransID();
 860     }
 861     return kInvalidTransID;
 862   }();
 863
 864   TCA start = genFuncPrologue(transID, kind, func, argc, code, fixups);
 865
 866   auto loc = maker.markEnd();
 867   if (RuntimeOption::EvalEnableReusableTC) {
 868     TCA UNUSED ms = loc.mainStart(), me = loc.mainEnd(),
 869                cs = loc.coldStart(), ce = loc.coldEnd(),
 870                fs = loc.frozenStart(), fe = loc.frozenEnd(),
 871                oldStart = start;
 872     bool did_relocate = relocateNewTranslation(loc, code, fixups, &start);
 873
 874     if (did_relocate) {
 875       FTRACE_MOD(reusetc, 1,
 876                  "Relocated prologue for func {} (id = {}) "
 877                  "from M[{}, {}], C[{}, {}], F[{}, {}] to M[{}, {}] "
 878                  "C[{}, {}] F[{}, {}] orig start @ {} new start @ {}\n",
 879                  func->fullName()->data(), func->getFuncId(),
 880                  ms, me, cs, ce, fs, fe, loc.mainStart(), loc.mainEnd(),
 881                  loc.coldStart(), loc.coldEnd(), loc.frozenStart(),
 882                  loc.frozenEnd(), oldStart, start);
 883     } else {
 884       FTRACE_MOD(reusetc, 1,
 885                  "Created prologue for func {} (id = {}) at "
 886                  "M[{}, {}], C[{}, {}], F[{}, {}] start @ {}\n",
 887                  func->fullName()->data(), func->getFuncId(),
 888                  ms, me, cs, ce, fs, fe, oldStart);
 889     }
 890
 891     recordFuncPrologue(func, loc);
 892     if (loc.mainStart() != aStart) {
 893       code.main().setFrontier(mainOrig); // we may have shifted to align
 894     }
 895   }
 896   if (RuntimeOption::EvalPerfRelocate) {
 897     GrowableVector<IncomingBranch> incomingBranches;
 898     recordPerfRelocMap(loc.mainStart(), loc.mainEnd(),
 899                        loc.coldCodeStart(), loc.coldEnd(),
 900                        funcBody, paramIndex,
 901                        incomingBranches,
 902                        fixups);
 903   }
 904   fixups.process(nullptr);
 905
 906   assertx(funcGuardMatches(funcGuardFromPrologue(start, func), func));
 907   assertx(m_code.isValidCodeAddress(start));
 908
 909   TRACE(2, "funcPrologue mcg %p %s(%d) setting prologue %p\n",
 910         this, func->fullName()->data(), argc, start);
 911   func->setPrologue(paramIndex, start);
 912
 913   assertx(kind == TransKind::LivePrologue ||
 914           kind == TransKind::ProfPrologue ||
 915           kind == TransKind::OptPrologue);
 916
 917   auto tr = maker.rec(funcBody, transID, kind);
 918   m_tx.addTranslation(tr);
 919   if (RuntimeOption::EvalJitUseVtuneAPI) {
 920     reportTraceletToVtune(func->unit(), func, tr);
 921   }
 922
 923
 924   recordGdbTranslation(funcBody, func, code.main(), aStart, false, true);
 925   recordBCInstr(OpFuncPrologue, loc.mainStart(), loc.mainEnd(), false);
 926
 927   m_numTrans++;
 928   assertx(m_numTrans <= RuntimeOption::EvalJitGlobalTranslationLimit);
 929
 930   return start;
 931 }
 932
 933 TCA MCGenerator::getFuncPrologue(Func* func, int nPassed, ActRec* ar,
 934                                  bool forRegeneratePrologue) {
 935   func->validate();
 936   TRACE(1, "funcPrologue %s(%d)\n", func->fullName()->data(), nPassed);
 937   int const numParams = func->numNonVariadicParams();
 938   int paramIndex = nPassed <= numParams ? nPassed : numParams + 1;
 939
 940   // Do a quick test before grabbing the write lease
 941   TCA prologue;
 942   if (checkCachedPrologue(func, paramIndex, prologue)) return prologue;
 943
 944   LeaseHolder writer(Translator::WriteLease());
 945   if (!writer.canWrite()) return nullptr;
 946
 947   // If we're regenerating a prologue, and we want to check shouldTranslate()
 948   // but ignore the code size limits.  We still want to respect the global
 949   // translation limit and other restrictions, though.
 950   if (forRegeneratePrologue) {
 951     if (!shouldTranslateNoSizeLimit(func)) return nullptr;
 952   } else {
 953     if (!shouldTranslate(func, TransKind::LivePrologue)) return nullptr;
 954   }
 955
 956   // Double check the prologue array now that we have the write lease
 957   // in case another thread snuck in and set the prologue already.
 958   if (checkCachedPrologue(func, paramIndex, prologue)) return prologue;
 959
 960   try {
 961     return emitFuncPrologue(func, nPassed, forRegeneratePrologue);
 962   } catch (const DataBlockFull& dbFull) {
 963
 964     // Fail hard if the block isn't code.hot.
 965     always_assert_flog(dbFull.name == "hot",
 966                        "data block = {}\nmessage: {}\n",
 967                        dbFull.name, dbFull.what());
 968
 969     // Otherwise, fall back to code.main and retry.
 970     m_code.disableHot();
 971     try {
 972       return emitFuncPrologue(func, nPassed, forRegeneratePrologue);
 973     } catch (const DataBlockFull& dbFull) {
 974       always_assert_flog(0, "data block = {}\nmessage: {}\n",
 975                          dbFull.name, dbFull.what());
 976     }
 977   }
 978 }
 979
 980 /**
 981  * Given the proflogueTransId for a TransProflogue translation, regenerate the
 982  * prologue (as a TransPrologue).  Returns the starting address for the
 983  * translation corresponding to triggerSk, if such translation is generated;
 984  * otherwise returns nullptr.
 985  */
 986 TCA MCGenerator::regeneratePrologue(TransID prologueTransId, SrcKey triggerSk,
 987                                     bool& emittedDVInit) {
 988   assertx(Translator::WriteLease().amOwner());
 989   auto rec = profData()->transRec(prologueTransId);
 990   auto func = rec->func();
 991   auto nArgs = rec->prologueArgs();
 992   emittedDVInit = false;
 993
 994   // Regenerate the prologue.
 995   func->resetPrologue(nArgs);
 996   auto const start = getFuncPrologue(
 997     func,
 998     nArgs,
 999     nullptr /* ActRec */,
1000     true /* regeneratePrologue */
1001   );
1002   if (!start) return nullptr;
1003
1004   func->setPrologue(nArgs, start);
1005
1006   // Smash callers of the old prologue with the address of the new one.
1007   for (auto toSmash : rec->mainCallers()) {
1008     smashCall(toSmash, start);
1009   }
1010
1011   // If the prologue has a matching guard, then smash its guard-callers as
1012   // well.
1013   auto const guard = funcGuardFromPrologue(start, func);
1014   if (funcGuardMatches(guard, func)) {
1015     for (auto toSmash : rec->guardCallers()) {
1016       smashCall(toSmash, guard);
1017     }
1018   }
1019   rec->clearAllCallers();
1020
1021   // If this prologue has a DV funclet, then generate a translation for the DV
1022   // funclet right after the prologue.
1023   TCA triggerSkStart = nullptr;
1024   if (nArgs < func->numNonVariadicParams()) {
1025     auto paramInfo = func->params()[nArgs];
1026     if (paramInfo.hasDefaultValue()) {
1027       SrcKey funcletSK(func, paramInfo.funcletOff, false);
1028       auto funcletTransId = profData()->dvFuncletTransId(func, nArgs);
1029       if (funcletTransId != kInvalidTransID) {
1030         invalidateSrcKey(funcletSK);
1031         auto args = TransArgs{funcletSK};
1032         args.transId = funcletTransId;
1033         args.kind = TransKind::Optimize;
1034         auto dvStart = translate(args).tca();
1035         emittedDVInit |= dvStart != nullptr;
1036         if (dvStart && !triggerSkStart && funcletSK == triggerSk) {
1037           triggerSkStart = dvStart;
1038         }
1039         // Flag that this translation has been retranslated, so that
1040         // it's not retranslated again along with the function body.
1041         profData()->setOptimized(funcletSK);
1042       }
1043     }
1044   }
1045
1046   return triggerSkStart;
1047 }
1048
1049 /**
1050  * Regenerate all prologues of func that were previously generated.
1051  * The prologues are sorted in ascending order of profile counters.
1052  * For prologues with corresponding DV funclets, their corresponding
1053  * DV funclet will be regenerated right after them.  The idea is to
1054  * generate the function body right after calling this function, so
1055  * that all prologues are placed right before it, and with the hottest
1056  * prologues closer to it.
1057  *
1058  * Returns the starting address for the translation corresponding to
1059  * triggerSk, if such translation is generated; otherwise returns
1060  * nullptr.
1061  */
1062 TCA MCGenerator::regeneratePrologues(Func* func, SrcKey triggerSk,
1063                                      bool& includedBody) {
1064   TCA triggerStart = nullptr;
1065   std::vector<TransID> prologTransIDs;
1066
1067   for (int nArgs = 0; nArgs < func->numPrologues(); nArgs++) {
1068     TransID tid = profData()->proflogueTransId(func, nArgs);
1069     if (tid != kInvalidTransID) {
1070       prologTransIDs.push_back(tid);
1071     }
1072   }
1073
1074   std::sort(prologTransIDs.begin(), prologTransIDs.end(),
1075           [&](TransID t1, TransID t2) -> bool {
1076             // This will sort in ascending order.
1077             return profData()->transCounter(t2) >
1078                    profData()->transCounter(t1);
1079           });
1080
1081   // Next, we're going to regenerate each prologue along with its DV funclet.
1082   // We consider the option of either including the DV funclets in the same
1083   // region as the function body or not.  Including them in the same region
1084   // enables some type information to flow and thus can eliminate some stores
1085   // and type checks, but it can also increase the code size by duplicating the
1086   // whole function body.  Therefore, we only include the function body along
1087   // with the DV init if both (a) the function has a single proflogue, and (b)
1088   // the size of the function is within a certain threshold.
1089   //
1090   // The mechanism used to keep the function body separate from the DV init is
1091   // to temporarily mark the SrcKey for the function body as already optimized.
1092   // (The region selectors break a region whenever they hit a SrcKey that has
1093   // already been optimized.)
1094   SrcKey funcBodySk(func, func->base(), false);
1095   includedBody = prologTransIDs.size() <= 1 &&
1096     func->past() - func->base() <= RuntimeOption::EvalJitPGOMaxFuncSizeDupBody;
1097
1098   if (!includedBody) profData()->setOptimized(funcBodySk);
1099   SCOPE_EXIT{ profData()->clearOptimized(funcBodySk); };
1100
1101   bool emittedAnyDVInit = false;
1102   for (TransID tid : prologTransIDs) {
1103     bool emittedDVInit = false;
1104     TCA start = regeneratePrologue(tid, triggerSk, emittedDVInit);
1105     if (triggerStart == nullptr && start != nullptr) {
1106       triggerStart = start;
1107     }
1108     emittedAnyDVInit |= emittedDVInit;
1109   }
1110
1111   // If we tried to include the function body along with a DV init, but didn't
1112   // end up generating any DV init, then flag that the function body was not
1113   // included.
1114   if (!emittedAnyDVInit) includedBody = false;
1115
1116   return triggerStart;
1117 }
1118
1119 /*
1120  * bindJmp --
1121  *
1122  *   Runtime service handler that patches a jmp to the translation of
1123  *   u:dest from toSmash.
1124  */
1125 TCA
1126 MCGenerator::bindJmp(TCA toSmash, SrcKey destSk, ServiceRequest req,
1127                      TransFlags trflags, bool& smashed) {
1128   auto args = TransArgs{destSk};
1129   args.flags = trflags;
1130   auto tDest = getTranslation(args).tca();
1131   if (!tDest) return nullptr;
1132
1133   LeaseHolder writer(Translator::WriteLease());
1134   if (!writer.canWrite()) return tDest;
1135
1136   auto const sr = m_srcDB.find(destSk);
1137   always_assert(sr);
1138   // The top translation may have changed while we waited for the write lease,
1139   // so read it again.  If it was replaced with a new translation, then bind to
1140   // the new one.  If it was invalidated, then don't bind the jump.
1141   tDest = sr->getTopTranslation();
1142   if (tDest == nullptr) return nullptr;
1143
1144   if (req == REQ_BIND_ADDR) {
1145     auto addr = reinterpret_cast<TCA*>(toSmash);
1146     if (*addr == tDest) {
1147       // Already smashed
1148       return tDest;
1149     }
1150     sr->chainFrom(IncomingBranch::addr(addr));
1151     smashed = true;
1152     return tDest;
1153   }
1154
1155   auto const isJcc = [&] {
1156     switch (arch()) {
1157       case Arch::X64: {
1158         x64::DecodedInstruction di(toSmash);
1159         return (di.isBranch() && !di.isJmp());
1160       }
1161
1162       case Arch::ARM: {
1163         using namespace vixl;
1164         struct JccDecoder : public Decoder {
1165           void VisitConditionalBranch(Instruction* inst) override {
1166             cc = true;
1167           }
1168           bool cc = false;
1169         };
1170         JccDecoder decoder;
1171         decoder.Decode(Instruction::Cast(toSmash));
1172         return decoder.cc;
1173       }
1174
1175       case Arch::PPC64:
1176         always_assert(false);
1177     }
1178     not_reached();
1179   }();
1180
1181   if (isJcc) {
1182     auto const target = smashableJccTarget(toSmash);
1183     assertx(target);
1184
1185     // Return if already smashed.
1186     if (target == tDest) return tDest;
1187     sr->chainFrom(IncomingBranch::jccFrom(toSmash));
1188   } else {
1189     auto const target = smashableJmpTarget(toSmash);
1190     assertx(target);
1191
1192     // Return if already smashed.
1193     if (!target || target == tDest) return tDest;
1194     sr->chainFrom(IncomingBranch::jmpFrom(toSmash));
1195   }
1196
1197   smashed = true;
1198   return tDest;
1199 }
1200
1201 namespace {
1202
1203 struct FreeRequestStubTrigger {
1204   explicit FreeRequestStubTrigger(TCA stub) : m_stub(stub) {
1205     TRACE(3, "FreeStubTrigger @ %p, stub %p\n", this, m_stub);
1206   }
1207   void operator()() {
1208     TRACE(3, "FreeStubTrigger: Firing @ %p , stub %p\n", this, m_stub);
1209     if (mcg->freeRequestStub(m_stub) != true) {
1210       // If we can't free the stub, enqueue again to retry.
1211       TRACE(3, "FreeStubTrigger: write lease failed, requeueing %p\n", m_stub);
1212       Treadmill::enqueue(FreeRequestStubTrigger(m_stub));
1213     }
1214   }
1215 private:
1216   TCA m_stub;
1217 };
1218
1219 }
1220
1221 void
1222 MCGenerator::enterTC(TCA start, ActRec* stashedAR) {
1223   if (debug) {
1224     fflush(stdout);
1225     fflush(stderr);
1226   }
1227
1228   assertx(m_code.isValidCodeAddress(start));
1229   assertx(((uintptr_t)vmsp() & (sizeof(Cell) - 1)) == 0);
1230   assertx(((uintptr_t)vmfp() & (sizeof(Cell) - 1)) == 0);
1231
1232   assertx(!Translator::WriteLease().amOwner());
1233
1234   INC_TPC(enter_tc);
1235   if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
1236     auto skData = SrcKey{liveFunc(), vmpc(), liveResumed()}.toAtomicInt();
1237     Trace::ringbufferEntry(RBTypeEnterTC, skData, (uint64_t)start);
1238   }
1239
1240   tl_regState = VMRegState::DIRTY;
1241   enterTCImpl(start, stashedAR);
1242   tl_regState = VMRegState::CLEAN;
1243   assertx(isValidVMStackAddress(vmsp()));
1244
1245   vmfp() = nullptr;
1246 }
1247
1248 TCA MCGenerator::handleServiceRequest(svcreq::ReqInfo& info) noexcept {
1249   FTRACE(1, "handleServiceRequest {}\n", svcreq::to_name(info.req));
1250
1251   assert_native_stack_aligned();
1252   tl_regState = VMRegState::CLEAN; // partially a lie: vmpc() isn't synced
1253
1254   if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
1255     Trace::ringbufferEntry(
1256       RBTypeServiceReq, (uint64_t)info.req, (uint64_t)info.args[0].tca
1257     );
1258   }
1259
1260   TCA start = nullptr;
1261   SrcKey sk;
1262   auto smashed = false;
1263
1264   switch (info.req) {
1265     case REQ_BIND_JMP:
1266     case REQ_BIND_ADDR: {
1267       auto const toSmash = info.args[0].tca;
1268       sk = SrcKey::fromAtomicInt(info.args[1].sk);
1269       auto const trflags = info.args[2].trflags;
1270       start = bindJmp(toSmash, sk, info.req, trflags, smashed);
1271       break;
1272     }
1273
1274     case REQ_RETRANSLATE: {
1275       INC_TPC(retranslate);
1276       sk = SrcKey{liveFunc(), info.args[0].offset, liveResumed()};
1277       auto trflags = info.args[1].trflags;
1278       auto args = TransArgs{sk};
1279       args.flags = trflags;
1280       start = retranslate(args).tca();
1281       SKTRACE(2, sk, "retranslated @%p\n", start);
1282       break;
1283     }
1284
1285     case REQ_RETRANSLATE_OPT: {
1286       sk = SrcKey::fromAtomicInt(info.args[0].sk);
1287       auto transID = info.args[1].transID;
1288       start = retranslateOpt(transID, false);
1289       SKTRACE(2, sk, "retranslated-OPT: transId = %d  start: @%p\n", transID,
1290               start);
1291       break;
1292     }
1293
1294     case REQ_POST_INTERP_RET: {
1295       // This is only responsible for the control-flow aspect of the Ret:
1296       // getting to the destination's translation, if any.
1297       auto ar = info.args[0].ar;
1298       auto caller = info.args[1].ar;
1299       assertx(caller == vmfp());
1300       // If caller is a resumable (aka a generator) then whats likely happened
1301       // here is that we're resuming a yield from. That expression happens to
1302       // cause an assumption that we made earlier to be violated (that `ar` is
1303       // on the stack), so if we detect this situation we need to fix up the
1304       // value of `ar`.
1305       if (UNLIKELY(caller->resumed() &&
1306                    caller->func()->isNonAsyncGenerator())) {
1307         auto gen = frame_generator(caller);
1308         if (gen->m_delegate.m_type == KindOfObject) {
1309           auto delegate = gen->m_delegate.m_data.pobj;
1310           // We only checked that our delegate is an object, but we can't get
1311           // into this situation if the object itself isn't a Generator
1312           assert(delegate->getVMClass() == Generator::getClass());
1313           // Ok so we're in a `yield from` situation, we know our ar is garbage.
1314           // The ar that we're looking for is the ar of the delegate generator,
1315           // so grab that here.
1316           ar = Generator::fromObject(delegate)->actRec();
1317         }
1318       }
1319       Unit* destUnit = caller->func()->unit();
1320       // Set PC so logging code in getTranslation doesn't get confused.
1321       vmpc() = destUnit->at(caller->m_func->base() + ar->m_soff);
1322       if (ar->isFCallAwait()) {
1323         // If there was an interped FCallAwait, and we return via the
1324         // jit, we need to deal with the suspend case here.
1325         assert(ar->m_r.m_aux.u_fcallAwaitFlag < 2);
1326         if (ar->m_r.m_aux.u_fcallAwaitFlag) {
1327           start = ustubs().fcallAwaitSuspendHelper;
1328           break;
1329         }
1330       }
1331       sk = SrcKey{caller->func(), vmpc(), caller->resumed()};
1332       start = getTranslation(TransArgs{sk}).tca();
1333       TRACE(3, "REQ_POST_INTERP_RET: from %s to %s\n",
1334             ar->m_func->fullName()->data(),
1335             caller->m_func->fullName()->data());
1336       break;
1337     }
1338
1339     case REQ_POST_DEBUGGER_RET: {
1340       auto fp = vmfp();
1341       auto caller = fp->func();
1342       vmpc() = caller->unit()->at(caller->base() +
1343                                   g_unwind_rds->debuggerReturnOff);
1344       FTRACE(3, "REQ_DEBUGGER_RET: pc {} in {}\n",
1345              vmpc(), fp->func()->fullName()->data());
1346       sk = SrcKey{fp->func(), vmpc(), fp->resumed()};
1347       start = getTranslation(TransArgs{sk}).tca();
1348       break;
1349     }
1350   }
1351
1352   if (smashed && info.stub) {
1353     Treadmill::enqueue(FreeRequestStubTrigger(info.stub));
1354   }
1355
1356   if (start == nullptr) {
1357     vmpc() = sk.unit()->at(sk.offset());
1358     start = ustubs().interpHelperSyncedPC;
1359   }
1360
1361   if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
1362     auto skData = sk.valid() ? sk.toAtomicInt() : uint64_t(-1LL);
1363     Trace::ringbufferEntry(RBTypeResumeTC, skData, (uint64_t)start);
1364   }
1365
1366   tl_regState = VMRegState::DIRTY;
1367   return start;
1368 }
1369
1370 TCA MCGenerator::handleBindCall(TCA toSmash,
1371                                 ActRec* calleeFrame,
1372                                 bool isImmutable) {
1373   Func* func = const_cast<Func*>(calleeFrame->m_func);
1374   int nArgs = calleeFrame->numArgs();
1375   TRACE(2, "bindCall %s, ActRec %p\n", func->fullName()->data(), calleeFrame);
1376   TCA start = getFuncPrologue(func, nArgs);
1377   TRACE(2, "bindCall -> %p\n", start);
1378   if (start && !isImmutable) {
1379     // We dont know we're calling the right function, so adjust start to point
1380     // to the dynamic check of ar->m_func.
1381     start = funcGuardFromPrologue(start, func);
1382   } else {
1383     TRACE(2, "bindCall immutably %s -> %p\n", func->fullName()->data(), start);
1384   }
1385
1386   if (start && !RuntimeOption::EvalFailJitPrologs) {
1387     LeaseHolder writer(Translator::WriteLease());
1388     if (writer.canWrite()) {
1389       // Someone else may have changed the func prologue while we waited for
1390       // the write lease, so read it again.
1391       start = getFuncPrologue(func, nArgs);
1392       if (start && !isImmutable) start = funcGuardFromPrologue(start, func);
1393
1394       if (start && smashableCallTarget(toSmash) != start) {
1395         assertx(smashableCallTarget(toSmash));
1396         TRACE(2, "bindCall smash %p -> %p\n", toSmash, start);
1397         smashCall(toSmash, start);
1398
1399         bool is_profiled = false;
1400         // For functions to be PGO'ed, if their current prologues are still
1401         // profiling ones (living in code.prof()), then save toSmash as a
1402         // caller to the prologue, so that it can later be smashed to call a
1403         // new prologue when it's generated.
1404         int calleeNumParams = func->numNonVariadicParams();
1405         int calledPrologNumArgs = (nArgs <= calleeNumParams ?
1406                                    nArgs :  calleeNumParams + 1);
1407         auto const profData = jit::profData();
1408         if (profData != nullptr && m_code.prof().contains(start)) {
1409           auto rec = profData->prologueTransRec(
1410             func,
1411             calledPrologNumArgs
1412           );
1413           if (isImmutable) {
1414             rec->addMainCaller(toSmash);
1415           } else {
1416             rec->addGuardCaller(toSmash);
1417           }
1418           is_profiled = true;
1419         }
1420
1421         // We need to be able to reclaim the function prologues once the unit
1422         // associated with this function is treadmilled-- so record all of the
1423         // callers that will need to be re-smashed
1424         if (RuntimeOption::EvalEnableReusableTC) {
1425           if (debug || !isImmutable) {
1426             recordFuncCaller(func, toSmash, isImmutable,
1427                              is_profiled, calledPrologNumArgs);
1428           }
1429         }
1430       }
1431     }
1432   } else {
1433     // We couldn't get a prologue address. Return a stub that will finish
1434     // entering the callee frame in C++, then call handleResume at the callee's
1435     // entry point.
1436     start = ustubs().fcallHelperThunk;
1437   }
1438
1439   return start;
1440 }
1441
1442 TCA MCGenerator::handleFCallAwaitSuspend() {
1443   assert_native_stack_aligned();
1444   FTRACE(1, "handleFCallAwaitSuspend\n");
1445
1446   tl_regState = VMRegState::CLEAN;
1447
1448   vmJitCalledFrame() = vmfp();
1449   SCOPE_EXIT { vmJitCalledFrame() = nullptr; };
1450
1451   auto start = suspendStack(vmpc());
1452   tl_regState = VMRegState::DIRTY;
1453   return start ? start : ustubs().resumeHelper;
1454 }
1455
1456 TCA MCGenerator::handleResume(bool interpFirst) {
1457   assert_native_stack_aligned();
1458   FTRACE(1, "handleResume({})\n", interpFirst);
1459
1460   if (!vmRegsUnsafe().pc) return ustubs().callToExit;
1461
1462   tl_regState = VMRegState::CLEAN;
1463
1464   auto sk = SrcKey{liveFunc(), vmpc(), liveResumed()};
1465   TCA start;
1466   if (interpFirst) {
1467     start = nullptr;
1468     INC_TPC(interp_bb_force);
1469   } else {
1470     start = getTranslation(TransArgs(sk)).tca();
1471   }
1472
1473   vmJitCalledFrame() = vmfp();
1474   SCOPE_EXIT { vmJitCalledFrame() = nullptr; };
1475
1476   // If we can't get a translation at the current SrcKey, interpret basic
1477   // blocks until we end up somewhere with a translation (which we may have
1478   // created, if the lease holder dropped it).
1479   while (!start) {
1480     INC_TPC(interp_bb);
1481     if (auto retAddr = HPHP::dispatchBB()) {
1482       start = retAddr;
1483       break;
1484     }
1485
1486     assertx(vmpc());
1487     sk = SrcKey{liveFunc(), vmpc(), liveResumed()};
1488     start = getTranslation(TransArgs{sk}).tca();
1489   }
1490
1491   if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
1492     auto skData = sk.valid() ? sk.toAtomicInt() : uint64_t(-1LL);
1493     Trace::ringbufferEntry(RBTypeResumeTC, skData, (uint64_t)start);
1494   }
1495
1496   tl_regState = VMRegState::DIRTY;
1497   return start;
1498 }
1499
1500 ///////////////////////////////////////////////////////////////////////////////
1501
1502 /*
1503  * Support for the stub freelist.
1504  */
1505 TCA FreeStubList::maybePop() {
1506   StubNode* ret = m_list;
1507   if (ret) {
1508     TRACE(1, "alloc stub %p\n", ret);
1509     m_list = ret->m_next;
1510     ret->m_freed = ~kStubFree;
1511   }
1512   return (TCA)ret;
1513 }
1514
1515 void FreeStubList::push(TCA stub) {
1516   /*
1517    * A freed stub may be released by Treadmill more than once if multiple
1518    * threads execute the service request before it is freed. We detect
1519    * duplicates by marking freed stubs
1520    */
1521   StubNode* n = reinterpret_cast<StubNode*>(stub);
1522   if (n->m_freed == kStubFree) {
1523     TRACE(1, "already freed stub %p\n", stub);
1524     return;
1525   }
1526   n->m_freed = kStubFree;
1527   n->m_next = m_list;
1528   TRACE(1, "free stub %p (-> %p)\n", stub, m_list);
1529   m_list = n;
1530 }
1531
1532 bool
1533 MCGenerator::freeRequestStub(TCA stub) {
1534   LeaseHolder writer(Translator::WriteLease());
1535   /*
1536    * If we can't acquire the write lock, the caller
1537    * (FreeRequestStubTrigger) retries
1538    */
1539   if (!writer.canWrite()) return false;
1540   assertx(m_code.frozen().contains(stub));
1541   m_debugInfo.recordRelocMap(stub, 0, "FreeStub");
1542   m_freeStubs.push(stub);
1543   return true;
1544 }
1545
1546 TCA MCGenerator::getFreeStub(CodeBlock& frozen, CGMeta* fixups,
1547                              bool* isReused) {
1548   TCA ret = m_freeStubs.maybePop();
1549   if (isReused) *isReused = ret;
1550
1551   if (ret) {
1552     Stats::inc(Stats::Astub_Reused);
1553     always_assert(m_freeStubs.peek() == nullptr ||
1554                   m_code.isValidCodeAddress(m_freeStubs.peek()));
1555     TRACE(1, "recycle stub %p\n", ret);
1556   } else {
1557     ret = frozen.frontier();
1558     Stats::inc(Stats::Astub_New);
1559     TRACE(1, "alloc new stub %p\n", ret);
1560   }
1561
1562   if (fixups) {
1563     fixups->reusedStubs.emplace_back(ret);
1564   }
1565   return ret;
1566 }
1567
1568 void MCGenerator::syncWork() {
1569   assertx(tl_regState != VMRegState::CLEAN);
1570   m_fixupMap.fixup(g_context.getNoCheck());
1571   tl_regState = VMRegState::CLEAN;
1572   Stats::inc(Stats::TC_Sync);
1573 }
1574
1575 namespace {
1576 /*
1577  * Attempt to emit code for the given IRUnit to `code'. Returns true on
1578  * success, false if codegen failed.
1579  */
1580 bool mcGenUnit(TransEnv& env, CodeCache::View code, CGMeta& fixups) {
1581   auto const& unit = *env.unit;
1582   try {
1583     emitVunit(*env.vunit, unit, code, fixups,
1584               dumpTCAnnotation(*env.args.sk.func(), env.args.kind)
1585               ? &env.annotations
1586               : nullptr);
1587   } catch (const DataBlockFull& dbFull) {
1588     if (dbFull.name == "hot") {
1589       mcg->code().disableHot();
1590       return false;
1591     } else {
1592       always_assert_flog(0, "data block = {}\nmessage: {}\n",
1593                          dbFull.name, dbFull.what());
1594     }
1595   }
1596
1597   auto const startSk = unit.context().srcKey();
1598   if (unit.context().kind == TransKind::Profile) {
1599     profData()->setProfiling(startSk.func()->getFuncId());
1600   }
1601
1602   return true;
1603 }
1604
1605 /*
1606  * If TC reuse is enabled, attempt to relocate the newly-emitted translation to
1607  * a hole reclaimed from dead code. Returns true if the translation was
1608  * relocated and false otherwise.
1609  */
1610 bool tryRelocateNewTranslation(SrcKey sk, TransLoc& loc,
1611                                CodeCache::View code, CGMeta& fixups) {
1612   if (!RuntimeOption::EvalEnableReusableTC) return false;
1613
1614   TCA UNUSED ms = loc.mainStart(), me = loc.mainEnd(),
1615              cs = loc.coldStart(), ce = loc.coldEnd(),
1616              fs = loc.frozenStart(), fe = loc.frozenEnd();
1617   bool did_relocate = relocateNewTranslation(loc, code, fixups);
1618
1619   if (did_relocate) {
1620     FTRACE_MOD(reusetc, 1,
1621                "Relocated translation for func {} (id = {})  @ sk({}) "
1622                "from M[{}, {}], C[{}, {}], F[{}, {}] to M[{}, {}] "
1623                "C[{}, {}] F[{}, {}]\n",
1624                sk.func()->fullName()->data(), sk.func()->getFuncId(),
1625                sk.offset(), ms, me, cs, ce, fs, fe, loc.mainStart(),
1626                loc.mainEnd(), loc.coldStart(), loc.coldEnd(),
1627                loc.frozenStart(), loc.frozenEnd());
1628   } else {
1629     FTRACE_MOD(reusetc, 1,
1630                "Created translation for func {} (id = {}) "
1631                " @ sk({}) at M[{}, {}], C[{}, {}], F[{}, {}]\n",
1632                sk.func()->fullName()->data(), sk.func()->getFuncId(),
1633                sk.offset(), ms, me, cs, ce, fs, fe);
1634   }
1635
1636   assertx(did_relocate == (loc.mainStart() != ms));
1637   return did_relocate;
1638 }
1639
1640 /*
1641  * If live code relocation is enabled, record metadata for the current
1642  * translation.
1643  */
1644 void recordRelocationMetaData(SrcKey sk, SrcRec& srcRec,
1645                               const TransLoc& loc, CGMeta& fixups) {
1646   if (!RuntimeOption::EvalPerfRelocate) return;
1647
1648   recordPerfRelocMap(loc.mainStart(), loc.mainEnd(),
1649                      loc.coldCodeStart(), loc.coldEnd(),
1650                      sk, -1,
1651                      srcRec.tailFallbackJumps(),
1652                      fixups);
1653 }
1654
1655 /*
1656  * If the jit maturity counter is enabled, update it with the current amount of
1657  * emitted code.
1658  */
1659 void reportJitMaturity(const CodeCache& code) {
1660   // Optimized translations are faster than profiling translations, which are
1661   // faster than the interpreter.  But when optimized translations are
1662   // generated, some profiling translations will become dead.  We assume the
1663   // incremental value of an optimized translation over the corresponding
1664   // profiling translations is comparable to the incremental value of a
1665   // profiling translation of similar size; thus we don't have to apply
1666   // different weights to code in different regions.
1667   auto const codeSize =
1668     code.hot().used() + code.main().used() + code.prof().used();
1669   if (s_jitMaturityCounter) {
1670     // EvalJitMatureSize is supposed to to be set to approximately 20% of the
1671     // code that will give us full performance, so recover the "fully mature"
1672     // size with some math.
1673     auto const fullSize = RuntimeOption::EvalJitMatureSize * 5;
1674     auto const after = codeSize >= fullSize ? 100
1675                                             : (codeSize * 100 / fullSize);
1676     auto const before = s_jitMaturityCounter->getValue();
1677     if (after > before) s_jitMaturityCounter->setValue(after);
1678   }
1679
1680   if (!s_loggedJitMature.load(std::memory_order_relaxed) &&
1681       StructuredLog::enabled() &&
1682       codeSize >= RuntimeOption::EvalJitMatureSize &&
1683       !s_loggedJitMature.exchange(true, std::memory_order_relaxed)) {
1684     StructuredLogEntry cols;
1685     cols.setInt("jit_mature_sec", time(nullptr) - HttpServer::StartTime);
1686     StructuredLog::log("hhvm_warmup", cols);
1687   }
1688 }
1689 }
1690
1691 TCA MCGenerator::finishTranslation(TransEnv env) {
1692   Timer timer(Timer::mcg_finishTranslation);
1693
1694   auto& args = env.args;
1695   auto const sk = args.sk;
1696
1697   profileSetHotFuncAttr();
1698
1699   // Grab the write lease before touching the TC. We use a blocking aqcuisition
1700   // because either a) we already have the lease and this will return right
1701   // away, or b) a caller above our pay grade decided to do the first part of
1702   // the translation process without the write lease, and wants us to make sure
1703   // we emit code here rather than throwing away the work already done.
1704   BlockingLeaseHolder write{Translator::WriteLease()};
1705   auto code = m_code.view(args.kind);
1706
1707   CGMeta fixups;
1708   TransLocMaker maker{code};
1709   maker.markStart();
1710
1711   if (env.vunit && !mcGenUnit(env, code, fixups)) {
1712     // mcGenUnit() failed. Roll back, drop the unit and region, and clear
1713     // fixups.
1714     maker.rollback();
1715     maker.markStart();
1716     env.unit.reset();
1717     env.vunit.reset();
1718     args.region.reset();
1719     fixups.clear();
1720   }
1721
1722   if (env.vunit) {
1723     m_numTrans++;
1724     assertx(m_numTrans <= RuntimeOption::EvalJitGlobalTranslationLimit);
1725   } else {
1726     args.kind = TransKind::Interp;
1727     FTRACE(1, "emitting dispatchBB interp request for failed "
1728            "translation (spOff = {})\n", env.initSpOffset.offset);
1729     vwrap(code.main(), code.data(), fixups,
1730           [&] (Vout& v) { emitInterpReq(v, sk, env.initSpOffset); },
1731           CodeKind::Helper);
1732   }
1733
1734   Timer metaTimer(Timer::mcg_finishTranslation_metadata);
1735   auto loc = maker.markEnd();
1736
1737   tryRelocateNewTranslation(sk, loc, code, fixups);
1738
1739   // Finally, record various metadata about the translation and add it to the
1740   // SrcRec.
1741   if (RuntimeOption::EvalProfileBC) {
1742     auto const vmUnit = sk.unit();
1743     TransBCMapping prev{};
1744     for (auto& cur : fixups.bcMap) {
1745       if (!cur.aStart) continue;
1746       if (prev.aStart) {
1747         if (prev.bcStart < vmUnit->bclen()) {
1748           recordBCInstr(uint32_t(vmUnit->getOp(prev.bcStart)),
1749                         prev.aStart, cur.aStart, false);
1750         }
1751       } else {
1752         recordBCInstr(OpTraceletGuard, loc.mainStart(), cur.aStart, false);
1753       }
1754       prev = cur;
1755     }
1756   }
1757
1758   auto const srcRec = m_srcDB.find(args.sk);
1759   always_assert(srcRec);
1760   recordRelocationMetaData(sk, *srcRec, loc, fixups);
1761   recordGdbTranslation(sk, sk.func(), code.main(), loc.mainStart(),
1762                        false, false);
1763   recordGdbTranslation(sk, sk.func(), code.cold(), loc.coldStart(),
1764                        false, false);
1765   if (args.kind == TransKind::Profile) {
1766     always_assert(args.region);
1767     profData()->addTransProfile(env.transID, args.region, env.pconds);
1768   }
1769
1770   auto tr = maker.rec(sk, env.transID, args.kind, args.region, fixups.bcMap,
1771                       std::move(env.annotations),
1772                       env.unit && cfgHasLoop(*env.unit));
1773   m_tx.addTranslation(tr);
1774   if (RuntimeOption::EvalJitUseVtuneAPI) {
1775     reportTraceletToVtune(sk.unit(), sk.func(), tr);
1776   }
1777
1778   GrowableVector<IncomingBranch> inProgressTailBranches;
1779   fixups.process(&inProgressTailBranches);
1780
1781   // SrcRec::newTranslation() makes this code reachable. Do this last;
1782   // otherwise there's some chance of hitting in the reader threads whose
1783   // metadata is not yet visible.
1784   TRACE(1, "newTranslation: %p  sk: (func %d, bcOff %d)\n",
1785         loc.mainStart(), sk.funcID(), sk.offset());
1786   srcRec->newTranslation(loc, inProgressTailBranches);
1787
1788   TRACE(1, "mcg: %zd-byte tracelet\n", (ssize_t)loc.mainSize());
1789   if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
1790     Trace::traceRelease("%s", getTCSpace().c_str());
1791   }
1792
1793   reportJitMaturity(m_code);
1794
1795   return loc.mainStart();
1796 }
1797
1798 MCGenerator::MCGenerator()
1799   : m_numTrans(0)
1800   , m_catchTraceMap(128)
1801 {
1802   TRACE(1, "MCGenerator@%p startup\n", this);
1803   mcg = this;
1804
1805   g_unwind_rds.bind();
1806
1807   static bool profileUp = false;
1808   if (!profileUp) {
1809     profileInit();
1810     profileUp = true;
1811   }
1812
1813   if (Trace::moduleEnabledRelease(Trace::printir) &&
1814       !RuntimeOption::EvalJit) {
1815     Trace::traceRelease("TRACE=printir is set but the jit isn't on. "
1816                         "Did you mean to run with -vEval.Jit=1?\n");
1817   }
1818
1819   s_jitMaturityCounter = ServiceData::createCounter("jit.maturity");
1820
1821   // Do not initialize JIT stubs for PPC64 - port under development
1822 #if !defined(__powerpc64__)
1823   m_ustubs.emitAll(m_code, m_debugInfo);
1824 #endif
1825
1826   // Write an .eh_frame section that covers the whole TC.
1827   EHFrameWriter ehfw;
1828   write_tc_cie(ehfw);
1829   ehfw.begin_fde(m_code.base());
1830   ehfw.end_fde(m_code.codeSize());
1831   ehfw.null_fde();
1832
1833   m_ehFrames.push_back(ehfw.register_and_release());
1834 }
1835
1836 folly::Optional<TCA> MCGenerator::getCatchTrace(CTCA ip) const {
1837   TCA* found = m_catchTraceMap.find(ip);
1838   if (found && *found != kInvalidCatchTrace) return *found;
1839   return folly::none;
1840 }
1841
1842 void codeEmittedThisRequest(size_t& requestEntry, size_t& now) {
1843   requestEntry = s_initialTCSize;
1844   now = mcg->code().totalUsed();
1845 }
1846
1847 namespace {
1848 __thread std::unordered_map<const ActRec*, TCA>* tl_debuggerCatches{nullptr};
1849 }
1850
1851 void stashDebuggerCatch(const ActRec* fp) {
1852   if (!tl_debuggerCatches) {
1853     tl_debuggerCatches = new std::unordered_map<const ActRec*, TCA>();
1854   }
1855
1856   auto optCatchBlock = mcg->getCatchTrace(TCA(fp->m_savedRip));
1857   always_assert(optCatchBlock && *optCatchBlock);
1858   auto catchBlock = *optCatchBlock;
1859   FTRACE(1, "Pushing debugger catch {} with fp {}\n", catchBlock, fp);
1860   tl_debuggerCatches->emplace(fp, catchBlock);
1861 }
1862
1863 TCA unstashDebuggerCatch(const ActRec* fp) {
1864   always_assert(tl_debuggerCatches);
1865   auto const it = tl_debuggerCatches->find(fp);
1866   always_assert(it != tl_debuggerCatches->end());
1867   auto const catchBlock = it->second;
1868   tl_debuggerCatches->erase(it);
1869   FTRACE(1, "Popped debugger catch {} for fp {}\n", catchBlock, fp);
1870   return catchBlock;
1871 }
1872
1873 void MCGenerator::requestInit() {
1874   tl_regState = VMRegState::CLEAN;
1875   Timer::RequestInit();
1876   memset(&tl_perf_counters, 0, sizeof(tl_perf_counters));
1877   Stats::init();
1878   requestInitProfData();
1879   s_initialTCSize = m_code.totalUsed();
1880 }
1881
1882 void MCGenerator::requestExit() {
1883   always_assert(!Translator::WriteLease().amOwner());
1884   TRACE_MOD(txlease, 2, "%" PRIx64 " write lease stats: %15" PRId64
1885             " kept, %15" PRId64 " grabbed\n",
1886             Process::GetThreadIdForTrace(), Translator::WriteLease().hintKept(),
1887             Translator::WriteLease().hintGrabbed());
1888   Stats::dump();
1889   Stats::clear();
1890   Timer::RequestExit();
1891   if (profData()) profData()->maybeResetCounters();
1892   requestExitProfData();
1893
1894   if (Trace::moduleEnabledRelease(Trace::mcgstats, 1)) {
1895     Trace::traceRelease("MCGenerator perf counters for %s:\n",
1896                         g_context->getRequestUrl(50).c_str());
1897     for (int i = 0; i < tpc_num_counters; i++) {
1898       Trace::traceRelease("%-20s %10" PRId64 "\n",
1899                           kPerfCounterNames[i], tl_perf_counters[i]);
1900     }
1901     Trace::traceRelease("\n");
1902   }
1903
1904   delete tl_debuggerCatches;
1905   tl_debuggerCatches = nullptr;
1906 }
1907
1908 MCGenerator::~MCGenerator() {
1909 }
1910
1911 static Debug::TCRange rangeFrom(const CodeBlock& cb, const TCA addr,
1912                                 bool isAcold) {
1913   assertx(cb.contains(addr));
1914   return Debug::TCRange(addr, cb.frontier(), isAcold);
1915 }
1916
1917 void MCGenerator::recordBCInstr(uint32_t op,
1918                                 const TCA addr,
1919                                 const TCA end,
1920                                 bool cold) {
1921   if (addr != end) {
1922     m_debugInfo.recordBCInstr(
1923       Debug::TCRange(addr, end, cold), op);
1924   }
1925 }
1926
1927 void MCGenerator::recordGdbTranslation(SrcKey sk,
1928                                        const Func* srcFunc,
1929                                        const CodeBlock& cb,
1930                                        const TCA start,
1931                                        bool exit,
1932                                        bool inPrologue) {
1933   if (start != cb.frontier()) {
1934     assertx(Translator::WriteLease().amOwner());
1935     if (!RuntimeOption::EvalJitNoGdb) {
1936       m_debugInfo.recordTracelet(rangeFrom(cb, start, &cb == &m_code.cold()),
1937                                  srcFunc,
1938                                  srcFunc->unit() ?
1939                                    srcFunc->unit()->at(sk.offset()) : nullptr,
1940                                  exit, inPrologue);
1941     }
1942     if (RuntimeOption::EvalPerfPidMap) {
1943       m_debugInfo.recordPerfMap(rangeFrom(cb, start, &cb == &m_code.cold()),
1944                                 srcFunc, exit, inPrologue);
1945     }
1946   }
1947 }
1948
1949 bool MCGenerator::addDbgGuards(const Unit* unit) {
1950   // TODO refactor
1951   // It grabs the write lease and iterates through whole SrcDB...
1952   struct timespec tsBegin, tsEnd;
1953   {
1954     BlockingLeaseHolder writer(Translator::WriteLease());
1955     auto code = m_code.view();
1956     auto& main = code.main();
1957     auto& data = code.data();
1958
1959     HPHP::Timer::GetMonotonicTime(tsBegin);
1960     // Doc says even find _could_ invalidate iterator, in pactice it should
1961     // be very rare, so go with it now.
1962     CGMeta fixups;
1963     for (auto& pair : m_srcDB) {
1964       SrcKey const sk = SrcKey::fromAtomicInt(pair.first);
1965       // We may have a SrcKey to a deleted function. NB: this may miss a
1966       // race with deleting a Func. See task #2826313.
1967       if (!Func::isFuncIdValid(sk.funcID())) continue;
1968       SrcRec* sr = pair.second;
1969       if (sr->unitMd5() == unit->md5() &&
1970           !sr->hasDebuggerGuard() &&
1971           m_tx.isSrcKeyInBL(sk)) {
1972         addDbgGuardImpl(sk, sr, main, data, fixups);
1973       }
1974     }
1975     fixups.process(nullptr);
1976   }
1977   HPHP::Timer::GetMonotonicTime(tsEnd);
1978   int64_t elapsed = gettime_diff_us(tsBegin, tsEnd);
1979   if (Trace::moduleEnabledRelease(Trace::mcg, 5)) {
1980     Trace::traceRelease("addDbgGuards got lease for %" PRId64 " us\n", elapsed);
1981   }
1982   return true;
1983 }
1984
1985 bool MCGenerator::addDbgGuard(const Func* func, Offset offset, bool resumed) {
1986   SrcKey sk(func, offset, resumed);
1987   {
1988     if (SrcRec* sr = m_srcDB.find(sk)) {
1989       if (sr->hasDebuggerGuard()) {
1990         return true;
1991       }
1992     } else {
1993       // no translation yet
1994       return true;
1995     }
1996   }
1997   if (debug) {
1998     if (!m_tx.isSrcKeyInBL(sk)) {
1999       TRACE(5, "calling addDbgGuard on PC that is not in blacklist");
2000       return false;
2001     }
2002   }
2003   BlockingLeaseHolder writer(Translator::WriteLease());
2004
2005   CGMeta fixups;
2006   if (SrcRec* sr = m_srcDB.find(sk)) {
2007     auto code = m_code.view();
2008     addDbgGuardImpl(sk, sr, code.main(), code.data(), fixups);
2009   }
2010   fixups.process(nullptr);
2011   return true;
2012 }
2013
2014 bool MCGenerator::dumpTCCode(const char* filename) {
2015 #define OPEN_FILE(F, SUFFIX)                                    \
2016   std::string F ## name = std::string(filename).append(SUFFIX); \
2017   FILE* F = fopen(F ## name .c_str(),"wb");                     \
2018   if (F == nullptr) return false;                               \
2019   SCOPE_EXIT{ fclose(F); };
2020
2021   OPEN_FILE(ahotFile,       "_ahot");
2022   OPEN_FILE(aFile,          "_a");
2023   OPEN_FILE(aprofFile,      "_aprof");
2024   OPEN_FILE(acoldFile,      "_acold");
2025   OPEN_FILE(afrozenFile,    "_afrozen");
2026   OPEN_FILE(helperAddrFile, "_helpers_addrs.txt");
2027
2028 #undef OPEN_FILE
2029
2030   // dump starting from the hot region
2031   auto result = true;
2032   auto writeBlock = [&](const CodeBlock& cb, FILE* file) {
2033     if (result) {
2034       auto const count = cb.used();
2035       result = fwrite(cb.base(), 1, count, file) == count;
2036     }
2037   };
2038
2039   writeBlock(m_code.hot(), ahotFile);
2040   writeBlock(m_code.main(), aFile);
2041   writeBlock(m_code.prof(), aprofFile);
2042   writeBlock(m_code.cold(), acoldFile);
2043   writeBlock(m_code.frozen(), afrozenFile);
2044   return result;
2045 }
2046
2047 bool MCGenerator::dumpTC(bool ignoreLease /* = false */) {
2048   folly::Optional<BlockingLeaseHolder> writer;
2049   if (!ignoreLease) {
2050     writer.emplace(Translator::WriteLease());
2051   }
2052   return dumpTCData() && dumpTCCode("/tmp/tc_dump");
2053 }
2054
2055 // Returns true on success
2056 bool MCGenerator::dumpTCData() {
2057   gzFile tcDataFile = gzopen("/tmp/tc_data.txt.gz", "w");
2058   if (!tcDataFile) return false;
2059
2060   if (!gzprintf(tcDataFile,
2061                 "repo_schema      = %s\n"
2062                 "ahot.base        = %p\n"
2063                 "ahot.frontier    = %p\n"
2064                 "a.base           = %p\n"
2065                 "a.frontier       = %p\n"
2066                 "aprof.base       = %p\n"
2067                 "aprof.frontier   = %p\n"
2068                 "acold.base       = %p\n"
2069                 "acold.frontier   = %p\n"
2070                 "afrozen.base     = %p\n"
2071                 "afrozen.frontier = %p\n\n",
2072                 repoSchemaId().begin(),
2073                 m_code.hot().base(), m_code.hot().frontier(),
2074                 m_code.main().base(), m_code.main().frontier(),
2075                 m_code.prof().base(), m_code.prof().frontier(),
2076                 m_code.cold().base(), m_code.cold().frontier(),
2077                 m_code.frozen().base(), m_code.frozen().frontier())) {
2078     return false;
2079   }
2080
2081   if (!gzprintf(tcDataFile, "total_translations = %zu\n\n",
2082                 m_tx.getCurrentTransID())) {
2083     return false;
2084   }
2085
2086   for (TransID t = 0; t < m_tx.getCurrentTransID(); t++) {
2087     if (gzputs(tcDataFile,
2088                m_tx.getTransRec(t)->print(m_tx.getTransCounter(t)).c_str()) ==
2089         -1) {
2090       return false;
2091     }
2092   }
2093
2094   gzclose(tcDataFile);
2095   return true;
2096 }
2097
2098 void MCGenerator::invalidateSrcKey(SrcKey sk) {
2099   assertx(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
2100   assertx(Translator::WriteLease().amOwner());
2101   /*
2102    * Reroute existing translations for SrcKey to an as-yet indeterminate
2103    * new one.
2104    */
2105   auto const sr = m_srcDB.find(sk);
2106   always_assert(sr);
2107   /*
2108    * Since previous translations aren't reachable from here, we know we
2109    * just created some garbage in the TC. We currently have no mechanism
2110    * to reclaim this.
2111    */
2112   FTRACE_MOD(reusetc, 1, "Replacing translations from func {} (id = {}) "
2113              "@ sk({}) in SrcRec addr={}\n",
2114              sk.func()->fullName()->data(), sk.func()->getFuncId(), sk.offset(),
2115              (void*)sr);
2116   Trace::Indent _i;
2117   sr->replaceOldTranslations();
2118 }
2119
2120 ///////////////////////////////////////////////////////////////////////////////
2121
2122 }}