hphp/runtime/vm/jit/tc-region.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/tc-region.h"
  18
  19 #include "hphp/runtime/vm/jit/tc.h"
  20 #include "hphp/runtime/vm/jit/tc-internal.h"
  21 #include "hphp/runtime/vm/jit/tc-prologue.h"
  22 #include "hphp/runtime/vm/jit/tc-record.h"
  23
  24 #include "hphp/runtime/base/perf-warning.h"
  25 #include "hphp/runtime/vm/jit/align.h"
  26 #include "hphp/runtime/vm/jit/cfg.h"
  27 #include "hphp/runtime/vm/jit/cg-meta.h"
  28 #include "hphp/runtime/vm/jit/code-cache.h"
  29 #include "hphp/runtime/vm/jit/func-order.h"
  30 #include "hphp/runtime/vm/jit/ir-unit.h"
  31 #include "hphp/runtime/vm/jit/irlower.h"
  32 #include "hphp/runtime/vm/jit/mcgen.h"
  33 #include "hphp/runtime/vm/jit/perf-counters.h"
  34 #include "hphp/runtime/vm/jit/prof-data.h"
  35 #include "hphp/runtime/vm/jit/relocation.h"
  36 #include "hphp/runtime/vm/jit/service-requests.h"
  37 #include "hphp/runtime/vm/jit/smashable-instr.h"
  38 #include "hphp/runtime/vm/jit/srcdb.h"
  39 #include "hphp/runtime/vm/jit/timer.h"
  40 #include "hphp/runtime/vm/jit/trans-db.h"
  41 #include "hphp/runtime/vm/jit/trans-rec.h"
  42 #include "hphp/runtime/vm/jit/translate-region.h"
  43 #include "hphp/runtime/vm/jit/vasm-emit.h"
  44 #include "hphp/runtime/vm/jit/vasm-gen.h"
  45 #include "hphp/runtime/vm/jit/vm-protect.h"
  46 #include "hphp/runtime/vm/jit/vtune-jit.h"
  47 #include "hphp/runtime/vm/resumable.h"
  48 #include "hphp/runtime/vm/treadmill.h"
  49
  50 #include "hphp/util/boot-stats.h"
  51 #include "hphp/util/service-data.h"
  52 #include "hphp/util/struct-log.h"
  53 #include "hphp/util/timer.h"
  54 #include "hphp/util/trace.h"
  55
  56 TRACE_SET_MOD(mcg);
  57
  58 namespace HPHP::jit::tc {
  59
  60 namespace {
  61
  62 using PrologueTCAMap = jit::hash_map<PrologueID,TCA,PrologueID::Hasher>;
  63 using SrcKeyTransMap = jit::hash_map<SrcKey,jit::vector<TCA>,
  64                                      SrcKey::Hasher>;
  65
  66 bool checkLimit(TransKind kind, const size_t numTrans) {
  67   auto const limit = kind == TransKind::Profile
  68     ? RuntimeOption::EvalJitMaxProfileTranslations
  69     : RuntimeOption::EvalJitMaxTranslations;
  70
  71   // Once numTrans has reached limit + 1 we know that an interp translation
  72   // has already been emitted. Prior to that if numTrans == limit only allow
  73   // interp translations to avoid a race where multiple threads believe there
  74   // are still available translations.
  75   if (numTrans == limit + 1) return false;
  76   if (numTrans == limit && kind != TransKind::Interp) return false;
  77   return true;
  78 }
  79
  80 void invalidateSrcKey(SrcKey sk, SBInvOffset spOff) {
  81   assertx(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
  82   /*
  83    * Reroute existing translations for SrcKey to an as-yet indeterminate
  84    * new one.
  85    */
  86   auto const sr = srcDB().find(sk);
  87   if (!sr) {
  88     always_assert(profData()->wasDeserialized());
  89     return;
  90   }
  91
  92   // Retranslate stub must exist, as createSrcRec() created it.
  93   auto const transStub = svcreq::getOrEmitStub(
  94     svcreq::StubType::Translate, sk, spOff);
  95   always_assert(transStub);
  96
  97   /*
  98    * Since previous translations aren't reachable from here, we know we
  99    * just created some garbage in the TC. We currently have no mechanism
 100    * to reclaim this.
 101    */
 102   FTRACE_MOD(Trace::reusetc, 1,
 103              "Replacing translations from sk: {} " "to SrcRec addr={}\n",
 104              showShort(sk), (void*)sr);
 105   Trace::Indent _i;
 106   sr->replaceOldTranslations(transStub);
 107 }
 108
 109 void invalidateFuncProfSrcKeys(const Func* func) {
 110   assertx(profData());
 111   auto const funcId = func->getFuncId();
 112   for (auto tid : profData()->funcProfTransIDs(funcId)) {
 113     auto const transRec = profData()->transRec(tid);
 114     invalidateSrcKey(transRec->srcKey(), transRec->startSpOff());
 115   }
 116 }
 117
 118 size_t infoSize(const FuncMetaInfo& info) {
 119   size_t sz = 0;
 120   for (auto& trans : info.translators) {
 121     auto const& range = trans->range();
 122     sz += range.main.size() + range.cold.size() + range.frozen.size();
 123   }
 124   return sz;
 125 }
 126
 127 bool checkTCLimits() {
 128   auto const main_under = code().main().used() < CodeCache::AMaxUsage;
 129   auto const cold_under = code().cold().used() < CodeCache::AColdMaxUsage;
 130   auto const froz_under = code().frozen().used() < CodeCache::AFrozenMaxUsage;
 131
 132   return main_under && cold_under && froz_under;
 133 }
 134
 135 void relocateOptFunc(FuncMetaInfo& info,
 136                      PrologueTCAMap& prologueTCAs,
 137                      SrcKeyTransMap& srcKeyTrans,
 138                      size_t* failedBytes = nullptr) {
 139   auto const func = info.func;
 140   unsigned nRegions = 0;
 141   // Relocate/emit all prologues and translations for func in order.
 142   for (auto& translator : info.translators) {
 143     assertx(func == translator->sk.func());
 144     auto const prologueTranslator =
 145       dynamic_cast<PrologueTranslator*>(translator.get());
 146     auto const regionTranslator =
 147       dynamic_cast<RegionTranslator*>(translator.get());
 148
 149     if (regionTranslator) {
 150       auto const it = srcKeyTrans.find(translator->sk);
 151       // We don't publish translations one at a time during optimized
 152       // retranslation.  Use the srcKeyTransMap to establish if we hit the
 153       // translation limit.
 154       if (it != srcKeyTrans.end() &&
 155           !checkLimit(regionTranslator->kind, it->second.size())) {
 156         translator->reset();
 157         continue;
 158       }
 159     }
 160     if (!translator->translateSuccess()) continue;
 161     auto const& range = translator->range();
 162     auto const bytes = range.main.size() + range.cold.size() +
 163                        range.frozen.size();
 164     if (regionTranslator) nRegions++;
 165     // We don't want to align the first region, to enable fall-through from the
 166     // last emitted prologue.
 167     const bool alignMain = !regionTranslator || nRegions != 1;
 168     translator->relocate(alignMain);
 169     if (!translator->translateSuccess()) {
 170       if (failedBytes) *failedBytes += bytes;
 171       continue;
 172     }
 173
 174     translator->bindOutgoingEdges();
 175     if (!translator->translateSuccess()) continue;
 176
 177     always_assert(code().inMain(translator->entry()));
 178     if (prologueTranslator) {
 179       const auto pid = PrologueID(func, prologueTranslator->paramIndex());
 180       prologueTCAs[pid] = translator->entry();
 181     } else if (regionTranslator) {
 182       srcKeyTrans[regionTranslator->sk].emplace_back(translator->entry());
 183     }
 184   }
 185 }
 186
 187 void publishOptFuncMeta(FuncMetaInfo& info) {
 188   for (auto const& translator : info.translators) {
 189     if (translator->translateSuccess()) translator->publishMetaInternal();
 190   }
 191 }
 192
 193 void publishOptFuncCode(FuncMetaInfo& info,
 194                         jit::hash_set<TCA>* publishedSet = nullptr) {
 195   auto const func = info.func;
 196
 197   // Publish all prologues and translations for func in order.
 198   for (auto const& translator : info.translators) {
 199     if (translator->translateSuccess()) {
 200       translator->publishCodeInternal();
 201       auto const tca = translator->entry();
 202       if (publishedSet) publishedSet->insert(tca);
 203       auto const numParams = func->numNonVariadicParams();
 204       if (translator->sk == SrcKey{func, numParams, SrcKey::FuncEntryTag{}} &&
 205           func->numRequiredParams() == numParams) {
 206         func->setFuncEntry(tca);
 207       }
 208     } else {
 209       // If we failed to emit the prologue (e.g. the TC filled up), redirect
 210       // all the callers to the fcallHelperThunk so that they stop calling
 211       // the profile code.
 212       translator->smashBackup();
 213     }
 214   }
 215 }
 216
 217 void relocateSortedOptFuncs(std::vector<FuncMetaInfo>& infos,
 218                             PrologueTCAMap& prologueTCAs,
 219                             SrcKeyTransMap& srcKeyTrans) {
 220   size_t failedBytes = 0;
 221
 222   bool shouldLog = RuntimeOption::ServerExecutionMode();
 223   for (auto& finfo : infos) {
 224     // We clear the translations that are not relocated to ensure
 225     // no one tries publishing such translations.
 226     if (!Func::isFuncIdValid(finfo.fid)) {
 227       finfo.clear();
 228       continue;
 229     }
 230
 231     // make sure we don't get ahead of the translation threads
 232     mcgen::waitForTranslate(finfo);
 233
 234     if (!checkTCLimits()) {
 235       FTRACE(1, "relocateSortedOptFuncs: ran out of space in the TC. "
 236              "Skipping function {} {}\n", finfo.func->getFuncId(),
 237              finfo.func->fullName());
 238       failedBytes += infoSize(finfo);
 239       // Reset the translators.  This will cause the callers of the OptPrologues
 240       // that are being skipped to be smashed to stop calling the corresponding
 241       // ProfPrologues (see publishOptFuncCode).
 242       for (auto& translator : finfo.translators) {
 243         translator->reset();
 244       }
 245       continue;
 246     }
 247     if (shouldLog) {
 248       shouldLog = false;
 249       Logger::Info("retranslateAll: starting to relocate functions");
 250     }
 251     relocateOptFunc(finfo, prologueTCAs, srcKeyTrans, &failedBytes);
 252   }
 253
 254   if (failedBytes) {
 255     FTRACE(1, "relocateSortedOptFuncs: failedBytes = {}\n", failedBytes);
 256     logPerfWarning("opt_translation_overflow", 1, [&] (StructuredLogEntry& e) {
 257       e.setInt("bytes_dropped", failedBytes);
 258     });
 259   }
 260 }
 261
 262 /*
 263  * Smash and optimize the calls in `meta' to prologues in `srcKeyTrans'.
 264  */
 265 void smashOptCalls(CGMeta& meta,
 266                    const PrologueTCAMap& prologueTCAs) {
 267   auto const oldSmashableCallData = std::move(meta.smashableCallData);
 268   for (auto& pair : oldSmashableCallData) {
 269     TCA call = pair.first;
 270     auto const& pid = pair.second;
 271     auto it = prologueTCAs.find(pid);
 272     if (it == prologueTCAs.end()) {
 273       // insert non-smashed call back into transInfo.meta.smashableCallData
 274       meta.smashableCallData.emplace(pair);
 275       continue;
 276     }
 277
 278     const TCA target = it->second;
 279     FTRACE(1, "smashedOptCalls: found candidate call @ {}, "
 280            "target prologue @ {} (funcId={}, nArgs={})\n",
 281            call, target, pid.funcId(), pid.nargs());
 282     assertx(code().inMainOrColdOrFrozen(call));
 283     assertx(code().inMainOrColdOrFrozen(target));
 284
 285     smashCall(call, target);
 286     optimizeSmashedCall(call);
 287
 288     meta.smashableLocations.erase(call);
 289   }
 290 }
 291
 292 /*
 293  * Find the jump target for jumps within the translation corresponding to
 294  * `curEntry' going to the SrcRec corresponding to `vec'.  `curEntry' is the TCA
 295  * of the start of the current tracelet.  For prologues this is nullptr.  It is
 296  * used while smashing retranslations of the same srckey.  `isRetrans` indicate
 297  * whether this is a fallback jump to the next translation.
 298  */
 299 TCA findJumpTarget(TCA curEntry,
 300                    const jit::vector<TCA>& vec,
 301                    bool isRetrans) {
 302   always_assert(vec.size() > 0);
 303
 304   // It may seem like !isRetrans should imply that none of the elements in the
 305   // retranslation chain vector is the curEntry.  In practice self loops may
 306   // make that happen.
 307   always_assert(IMPLIES(curEntry == nullptr, !isRetrans));
 308
 309   // Case 1: when the jump is in a prologue (curEntry == nullptr) or it jumps to a
 310   //         different SrcKey, we jump to the first translation in the list.
 311   if (!isRetrans) {
 312     return vec.front();
 313   }
 314
 315   // Case 2: when jumping to the same SrcKey, we jump to the translation
 316   //         following this one (`curEntry').
 317   for (size_t i = 0; i < vec.size() - 1; i++) {
 318     if (vec[i] == curEntry) {
 319       return vec[i + 1];
 320     }
 321   }
 322
 323   // We should only get here if `curEntry' is the last translation.
 324   always_assert(curEntry == vec.back());
 325   return nullptr;
 326 }
 327
 328 /*
 329  * Smash and optimize the jumps in the translation starting at TCA entry (with
 330  * tracked metadata in meta)  going to the optimized translations in
 331  * `srcKeyTrans'.
 332  */
 333 void smashOptBinds(CGMeta& meta,
 334                    TCA entry,  // nullptr for prologues
 335                    const SrcKeyTransMap& srcKeyTrans) {
 336   jit::hash_set<TCA> smashed;
 337   decltype(meta.smashableBinds) newBinds;
 338
 339   for (auto& bind : meta.smashableBinds) {
 340     auto it = srcKeyTrans.find(bind.sk);
 341     if (it == srcKeyTrans.end()) {
 342       newBinds.emplace_back(bind);
 343       continue;
 344     }
 345     const auto& transVec = it->second;
 346     TCA succTCA = findJumpTarget(entry, transVec, bind.fallback);
 347     if (succTCA == nullptr) {
 348       newBinds.emplace_back(bind);
 349       continue;
 350     }
 351
 352     FTRACE(3, "smashOptBinds: found candidate bind @ {}, target SrcKey {}, "
 353            "fallback = {}\n",
 354            bind.smashable.show(), showShort(bind.sk), bind.fallback);
 355     // The bound ADDR pointer may not belong to the data segment, as is the case
 356     // with SSwitchMap (see #10347945)
 357     assertx(code().inMainOrColdOrFrozen(bind.smashable.toSmash()) ||
 358             bind.smashable.type() == IncomingBranch::Tag::ADDR);
 359     assertx(code().inMainOrColdOrFrozen(succTCA));
 360     bind.smashable.patch(succTCA);
 361     bind.smashable.optimize();
 362     smashed.insert(bind.smashable.toSmash());
 363     meta.smashableLocations.erase(bind.smashable.toSmash());
 364   }
 365
 366   // Remove jumps that were smashed from meta.smashableJumpData.
 367   meta.smashableBinds.swap(newBinds);
 368
 369   // If any of the jumps we smashed was a tail jump (i.e. going to a
 370   // retranslation of the entry SrcKey), then remove it from the set of
 371   // inProgressTailJumps as it has already been smashed.
 372   GrowableVector<IncomingBranch> newTailJumps;
 373   for (auto& jump : meta.inProgressTailJumps) {
 374     if (smashed.count(jump.toSmash()) == 0) {
 375       newTailJumps.push_back(jump);
 376     } else {
 377       FTRACE(3, "smashOptBinds: removing {} from inProgressTailJumps\n",
 378              jump.toSmash());
 379     }
 380   }
 381   meta.inProgressTailJumps.swap(newTailJumps);
 382 }
 383
 384 /*
 385  * Smash and optimize the calls and jumps between the translations/prologues in
 386  * `infos'.
 387  */
 388 void smashOptSortedOptFuncs(std::vector<FuncMetaInfo>& infos,
 389                             const PrologueTCAMap& prologueTCAs,
 390                             const SrcKeyTransMap& srcKeyTrans) {
 391   BootStats::Block timer("RTA_smash_opt_funcs",
 392                          RuntimeOption::ServerExecutionMode());
 393   for (auto& finfo : infos) {
 394     if (!Func::isFuncIdValid(finfo.fid)) continue;
 395
 396     for (auto& translator : finfo.translators) {
 397       // Skip if the translation wasn't relocated (e.g. ran out of TC space).
 398       if (!translator->entry()) continue;
 399       assertx(code().inMainOrColdOrFrozen(translator->entry()));
 400
 401       if (isPrologue(translator->kind)) {
 402         smashOptBinds(translator->meta(), nullptr, srcKeyTrans);
 403       } else {
 404         smashOptCalls(translator->meta(), prologueTCAs);
 405         smashOptBinds(translator->meta(), translator->entry(), srcKeyTrans);
 406       }
 407     }
 408   }
 409 }
 410
 411 void invalidateFuncsProfSrcKeys() {
 412   BootStats::Block timer("RTA_invalidate_prof_srckeys",
 413                          RuntimeOption::ServerExecutionMode());
 414   auto const pd = profData();
 415   assertx(pd);
 416   pd->forEachProfilingFunc([&](auto const& func) {
 417     always_assert(func);
 418
 419     invalidateFuncProfSrcKeys(func);
 420
 421     // clear the func body and prologues
 422     const_cast<Func*>(func)->resetFuncEntry();
 423     auto const numPrologues = func->numPrologues();
 424     for (int p = 0; p < numPrologues; p++) {
 425       const_cast<Func*>(func)->resetPrologue(p);
 426     }
 427   });
 428 }
 429
 430 void publishSortedOptFuncsMeta(std::vector<FuncMetaInfo>& infos) {
 431   BootStats::Block timer("RTA_publish_meta",
 432                          RuntimeOption::ServerExecutionMode());
 433   for (auto& finfo : infos) {
 434     if (Func::isFuncIdValid(finfo.fid)) {
 435       publishOptFuncMeta(finfo);
 436     }
 437   }
 438 }
 439
 440 void publishSortedOptFuncsCode(std::vector<FuncMetaInfo>& infos,
 441                                jit::hash_set<TCA>* publishedSet) {
 442   BootStats::Block timer("RTA_publish_code",
 443                          RuntimeOption::ServerExecutionMode());
 444   for (auto& finfo : infos) {
 445     if (Func::isFuncIdValid(finfo.fid)) {
 446       publishOptFuncCode(finfo, publishedSet);
 447     }
 448   }
 449 }
 450
 451 std::string show(const SrcKeyTransMap& map) {
 452   std::string ret;
 453   for (auto& skt : map) {
 454     folly::format(&ret, "  - [{}]:", showShort(skt.first));
 455     for (auto tca : skt.second) {
 456       folly::format(&ret, " {},", tca);
 457     }
 458     ret += "\n";
 459   }
 460   return ret;
 461 }
 462
 463 void checkPublishedAddr(TCA tca, const jit::hash_set<TCA>& publishedSet) {
 464   always_assert_flog(code().inMainOrColdOrFrozen(tca),
 465                      "srcKeyTrans has address not in hot/main: {}", tca);
 466   always_assert_flog(publishedSet.count(tca),
 467                      "srcKeyTrans has unpublished translation @ {}", tca);
 468 }
 469
 470 /*
 471  * Make sure that every address that we may have smashed to was published
 472  * somewhere in the translation cache.
 473  */
 474 void checkPublishedAddresses(const PrologueTCAMap&     prologueTCAs,
 475                              const SrcKeyTransMap&     srcKeyTrans,
 476                              const jit::hash_set<TCA>& publishedSet) {
 477   for (auto& prologueTCA : prologueTCAs) {
 478       checkPublishedAddr(prologueTCA.second, publishedSet);
 479   }
 480
 481   for (auto& skt : srcKeyTrans) {
 482     auto& vec = skt.second;
 483     for (auto tca : vec) {
 484       if (tca) {
 485         checkPublishedAddr(tca, publishedSet);
 486       }
 487     }
 488   }
 489 }
 490
 491 ////////////////////////////////////////////////////////////////////////////////
 492 }
 493
 494 SrcRec* findSrcRec(SrcKey sk) {
 495   return srcDB().find(sk);
 496 }
 497
 498 SrcRec* createSrcRec(SrcKey sk, SBInvOffset spOff) {
 499   if (auto const sr = srcDB().find(sk)) return sr;
 500
 501   // invalidateSrcKey() depends on existence of this stub.
 502   auto const transStub = svcreq::getOrEmitStub(
 503     svcreq::StubType::Translate, sk, spOff);
 504   if (!transStub) return nullptr;
 505
 506   auto metaLock = lockMetadata();
 507   if (auto const sr = srcDB().find(sk)) return sr;
 508
 509   SKTRACE(1, sk, "inserting SrcRec\n");
 510
 511   auto const sr = srcDB().insert(sk);
 512   if (RuntimeOption::EvalEnableReusableTC) {
 513     recordFuncSrcRec(sk.func(), sr);
 514   }
 515
 516   return sr;
 517 }
 518
 519
 520 ////////////////////////////////////////////////////////////////////////////////
 521
 522 void publishOptFunc(FuncMetaInfo info) {
 523   PrologueTCAMap prologueTCAs;
 524   SrcKeyTransMap srcKeyTrans;
 525   relocateOptFunc(info, prologueTCAs, srcKeyTrans);
 526
 527   auto codeLock = lockCode();
 528   auto metaLock = lockMetadata();
 529   invalidateFuncProfSrcKeys(Func::fromFuncId(info.fid));
 530   publishOptFuncMeta(info);
 531   publishOptFuncCode(info);
 532
 533   updateCodeSizeCounters();
 534 }
 535
 536 void relocatePublishSortedOptFuncs(std::vector<FuncMetaInfo> infos) {
 537   // Grab the session now (which includes a Treadmill::Session) so
 538   // that no Func's get destroyed during this step
 539   ProfData::Session pds(Treadmill::SessionKind::RetranslateAll);
 540   const bool serverMode = RuntimeOption::ServerExecutionMode();
 541
 542   PrologueTCAMap prologueTCAs;
 543   SrcKeyTransMap srcKeyTrans;
 544
 545   relocateSortedOptFuncs(infos, prologueTCAs, srcKeyTrans);
 546
 547   if (serverMode) {
 548     Logger::Info(
 549       "retranslateAll: finished optimizing and relocating functions"
 550     );
 551   }
 552
 553   auto codeLock = lockCode();
 554   auto metaLock = lockMetadata();
 555
 556   FTRACE(3,
 557          "relocatePublishSortedOptFuncs: after relocateSortedOptFuncs:\n{}\n",
 558          show(srcKeyTrans));
 559
 560   smashOptSortedOptFuncs(infos, prologueTCAs, srcKeyTrans);
 561
 562   FTRACE(3,
 563          "relocatePublishSortedOptFuncs: after smashOptSortedOptFuncs:\n{}\n",
 564          show(srcKeyTrans));
 565
 566   if (serverMode) {
 567     Logger::Info("retranslateAll: starting to publish functions");
 568   }
 569
 570   invalidateFuncsProfSrcKeys();
 571
 572   // Publish the metadata for all the translations/prologues before actually
 573   // publishing any of their code.  This is necessary because we've smashed
 574   // calls and jumps across translations so, once any of them is made reachable,
 575   // we have to assume that all of them are already reachable, even before
 576   // they're all published.
 577   publishSortedOptFuncsMeta(infos);
 578
 579   jit::hash_set<TCA> publishedSet;
 580   publishSortedOptFuncsCode(infos, debug ? &publishedSet : nullptr);
 581
 582   FTRACE(3,
 583          "relocatePublishSortedOptFuncs: after publishSortedOptFuncsCode:\n"
 584          "{}\n", show(srcKeyTrans));
 585
 586   if (debug) {
 587     checkPublishedAddresses(prologueTCAs, srcKeyTrans, publishedSet);
 588   }
 589
 590   updateCodeSizeCounters();
 591 }
 592
 593 void RegionTranslator::computeKind() {
 594   // Update the translation kind if it is invalid, or if it may
 595   // have changed (original kind was a profiling kind)
 596   if (kind == TransKind::Invalid || kind == TransKind::Profile) {
 597     kind = profileFunc(sk.func()) ? TransKind::Profile
 598                                   : TransKind::Live;
 599   }
 600 }
 601
 602 Optional<TranslationResult> RegionTranslator::getCached() {
 603   auto const srcRec = srcDB().find(sk);
 604   auto const numTrans = srcRec->numTrans();
 605   if (prevNumTranslations != -1 && prevNumTranslations != numTrans) {
 606     // A new translation was generated before we grabbed the lock.  Force
 607     // execution to rerun through the retranslation chain.
 608     return TranslationResult{srcRec->getTopTranslation()};
 609   }
 610   prevNumTranslations = numTrans;
 611
 612   // An optimize retranslation request may be in flight that will
 613   // remove profiling translations from the srcRec.  Wait till
 614   // getCached is called while the lease is held to check the
 615   // number of translations in the srcRec is not over max
 616   // capacity.
 617   if (!m_lease || !(*m_lease)) return std::nullopt;
 618   // Check for potential interp anchor translation
 619   if (kind == TransKind::Profile) {
 620     if (numTrans > RuntimeOption::EvalJitMaxProfileTranslations) {
 621       always_assert(numTrans ==
 622                     RuntimeOption::EvalJitMaxProfileTranslations + 1);
 623       return TranslationResult{srcRec->getTopTranslation()};
 624     }
 625   } else if (numTrans > RuntimeOption::EvalJitMaxTranslations) {
 626     always_assert(numTrans == RuntimeOption::EvalJitMaxTranslations + 1);
 627     return TranslationResult{srcRec->getTopTranslation()};
 628   }
 629   return std::nullopt;
 630 }
 631
 632 void RegionTranslator::resetCached() {
 633   invalidateSrcKey(sk, spOff);
 634 }
 635
 636 void RegionTranslator::gen() {
 637   auto const srcRec = srcDB().find(sk);
 638   auto const emitInterpStub = [&] {
 639     FTRACE(1, "emitting dispatchBB interp request for failed "
 640            "translation (spOff = {})\n", spOff.offset);
 641     vunit = std::make_unique<Vunit>();
 642     Vout vmain{*vunit, vunit->makeBlock(AreaIndex::Main)};
 643     vunit->entry = Vlabel(vmain);
 644
 645     emitInterpReq(vmain, sk, spOff);
 646
 647     irlower::optimize(*vunit, CodeKind::Helper);
 648   };
 649   auto const fail = [&] {
 650     if (isProfiling(kind)) return;
 651
 652     kind = TransKind::Interp;
 653     if (!checkLimit(kind, srcRec->numTrans())) return;
 654     emitInterpStub();
 655   };
 656
 657   // Only start profiling new functions at their entry point. This reduces the
 658   // chances of profiling the body of a function but not its entry (where we
 659   // trigger retranslation) and helps remove bias towards larger functions that
 660   // can cause variations in the size of code.prof.
 661   if (kind == TransKind::Profile &&
 662       !profData()->profiling(sk.funcID()) &&
 663       !sk.funcEntry()) {
 664     return;
 665   }
 666   if (!checkLimit(kind, srcRec->numTrans())) return fail();
 667
 668   if (!region) {
 669     region = selectRegion(regionContext(), kind);
 670   }
 671   if (!region) return fail();
 672
 673   INC_TPC(translate);
 674   Timer timer(Timer::mcg_translate);
 675
 676   tracing::Block _{"translate", [&] {
 677                 return traceProps(sk.func())
 678                         .add("sk", show(sk))
 679                         .add("trans_kind", show(kind));
 680         }};
 681   tracing::annotateBlock(
 682     [&] {
 683       return tracing::Props{}
 684       .add("region_size", region->instrSize());
 685     }
 686   );
 687
 688   rqtrace::ScopeGuard trace{"JIT_TRANSLATE"};
 689   trace.annotate("func_name", sk.func()->fullName()->data());
 690   trace.annotate("trans_kind", show(kind));
 691   trace.setEventPrefix("JIT_");
 692   trace.annotate(
 693     "region_size", folly::to<std::string>(region->instrSize()));
 694
 695   TransContext ctx {
 696     transId == kInvalidTransID ? TransIDSet{} : TransIDSet{transId},
 697     optIndex,
 698     kind,
 699     sk,
 700     region.get(),
 701     PrologueID(),
 702   };
 703   unit = irGenRegion(*region, ctx, pconds);
 704   assertx(unit);
 705   auto const& uas = unit->annotationData->getAllAnnotations();
 706   annotations.insert(annotations.end(), uas.begin(), uas.end());
 707   hasLoop = cfgHasLoop(*unit);
 708
 709   vunit = irlower::lowerUnit(*unit);
 710   // vasm gen can punt, and when it does it's important we don't retry
 711   // translating the same thing again and again as it will potentially consume
 712   // more and more RDS space.
 713   if (!vunit) emitInterpStub();
 714 }
 715
 716 /*
 717  * Record various metadata about the translation into the global data
 718  * structures.
 719  */
 720 void RegionTranslator::publishMetaImpl() {
 721   auto& fixups = transMeta->fixups;
 722   const auto& view = transMeta->view;
 723   const auto& loc = transMeta->range.loc();
 724   assertx(!loc.empty());
 725
 726   auto const srcRec = srcDB().find(sk);
 727   always_assert(srcRec);
 728
 729   if (RuntimeOption::EvalProfileBC) {
 730     TransBCMapping prev{};
 731     for (auto& cur : fixups.bcMap) {
 732       if (!cur.aStart) continue;
 733       if (prev.aStart) {
 734         recordBCInstr(uint32_t(prev.sk.op()), prev.aStart, cur.aStart, false);
 735       } else {
 736         recordBCInstr(OpTraceletGuard, loc.mainStart(), cur.aStart, false);
 737       }
 738       prev = cur;
 739     }
 740   }
 741
 742   recordGdbTranslation(sk, view.main(), loc.mainStart(), loc.mainEnd());
 743   recordGdbTranslation(sk, view.cold(), loc.coldCodeStart(), loc.coldEnd());
 744
 745   TransRec tr{sk, transId, kind, loc.mainStart(), loc.mainSize(),
 746               loc.coldCodeStart(), loc.coldCodeSize(),
 747               loc.frozenCodeStart(), loc.frozenCodeSize(),
 748               std::move(annotations), region, fixups.bcMap, hasLoop};
 749   transdb::addTranslation(tr);
 750   FuncOrder::recordTranslation(tr);
 751   if (RuntimeOption::EvalJitUseVtuneAPI) {
 752     reportTraceletToVtune(sk.unit(), sk.func(), tr);
 753   }
 754   recordTranslationSizes(tr);
 755
 756   fixups.process(&tailBranches);
 757 }
 758
 759 /*
 760  * Add a translation to the corresponding SrcRec, effectively making it
 761  * reachable.  This should be done after the metadata for the translation has
 762  * been published.
 763  */
 764 void RegionTranslator::publishCodeImpl() {
 765   const auto& loc = transMeta->range.loc();
 766   const auto srcRec = srcDB().find(sk);
 767   always_assert(srcRec);
 768   assertx(checkLimit(kind, srcRec->numTrans()));
 769
 770   if (kind == TransKind::Profile) {
 771     always_assert(region);
 772     profData()->addTransProfile(transId, region, pconds,
 773                                 transMeta->range.main.size());
 774   }
 775
 776   TRACE(1, "newTranslation: %p  sk: %s\n",
 777         entry(), showShort(sk).c_str());
 778
 779   srcRec->newTranslation(loc, tailBranches);
 780
 781   TRACE(1, "mcg: %u-byte translation (%u main, %u cold, %u frozen)\n",
 782         loc.mainSize() + loc.coldCodeSize() + loc.frozenCodeSize(),
 783         loc.mainSize(), loc.coldCodeSize(), loc.frozenCodeSize());
 784   if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
 785     Trace::traceRelease("%s", getTCSpace().c_str());
 786   }
 787   checkFreeProfData();
 788 }
 789
 790 void RegionTranslator::setCachedForProcessFail() {
 791   auto const srcRec = srcDB().find(sk);
 792   always_assert(srcRec);
 793
 794   auto const stub = svcreq::emit_interp_no_translate_stub(spOff, sk);
 795   TRACE(1, "setCachedForProcessFail: stub: %p  sk: %s\n",
 796         stub, showShort(sk).c_str());
 797
 798   if (!stub) return;
 799   srcRec->smashFallbacksToStub(stub);
 800 }
 801
 802 }