hphp/runtime/vm/jit/translate-region.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/translate-region.h"
  18
  19 #include "hphp/util/map-walker.h"
  20 #include "hphp/util/ringbuffer.h"
  21 #include "hphp/util/timer.h"
  22 #include "hphp/util/trace.h"
  23
  24 #include "hphp/runtime/base/arch.h"
  25 #include "hphp/runtime/base/runtime-option.h"
  26
  27 #include "hphp/runtime/vm/bc-pattern.h"
  28
  29 #include "hphp/runtime/vm/jit/ir-unit.h"
  30 #include "hphp/runtime/vm/jit/inlining-decider.h"
  31 #include "hphp/runtime/vm/jit/irgen.h"
  32 #include "hphp/runtime/vm/jit/mc-generator.h"
  33 #include "hphp/runtime/vm/jit/normalized-instruction.h"
  34 #include "hphp/runtime/vm/jit/print.h"
  35 #include "hphp/runtime/vm/jit/prof-data.h"
  36 #include "hphp/runtime/vm/jit/punt.h"
  37 #include "hphp/runtime/vm/jit/region-selection.h"
  38 #include "hphp/runtime/vm/jit/timer.h"
  39 #include "hphp/runtime/vm/jit/type.h"
  40
  41 TRACE_SET_MOD(trans);
  42
  43 namespace HPHP { namespace jit {
  44
  45 //////////////////////////////////////////////////////////////////////
  46
  47 namespace {
  48
  49 /*
  50  * Create a map from RegionDesc::BlockId -> IR Block* for all region blocks.
  51  */
  52 BlockIdToIRBlockMap createBlockMap(IRGS& irgs, const RegionDesc& region) {
  53   auto ret = BlockIdToIRBlockMap{};
  54
  55   auto& irb = *irgs.irb;
  56   auto const& blocks = region.blocks();
  57   for (unsigned i = 0; i < blocks.size(); i++) {
  58     auto const rBlock = blocks[i];
  59     auto const id = rBlock->id();
  60
  61     // NB: This maps the region entry block to a new IR block, even though
  62     // we've already constructed an IR entry block. We'll make the IR entry
  63     // block jump to this block.
  64     auto const iBlock = irb.unit().defBlock();
  65
  66     ret[id] = iBlock;
  67     FTRACE(1,
  68            "createBlockMaps: RegionBlock {} => IRBlock {} (BC offset = {})\n",
  69            id, iBlock->id(), rBlock->start().offset());
  70   }
  71
  72   return ret;
  73 }
  74
  75 /*
  76  * Set IRBuilder's Block associated to blockId's block according to
  77  * the mapping in blockIdToIRBlock.
  78  */
  79 void setIRBlock(IRGS& irgs,
  80                 RegionDesc::BlockId blockId,
  81                 const RegionDesc& region,
  82                 const BlockIdToIRBlockMap& blockIdToIRBlock) {
  83   auto& irb = *irgs.irb;
  84   auto rBlock = region.block(blockId);
  85   auto sk = rBlock->start();
  86
  87   auto iit = blockIdToIRBlock.find(blockId);
  88   assertx(iit != blockIdToIRBlock.end());
  89
  90   assertx(!irb.hasBlock(sk));
  91   FTRACE(3, "  setIRBlock: blockId {}, offset {} => IR Block {}\n",
  92          blockId, sk.offset(), iit->second->id());
  93   irb.setBlock(sk, iit->second);
  94 }
  95
  96 /*
  97  * Set IRBuilder's Blocks for srcBlockId's successors' offsets within
  98  * the region.  It also sets the guard-failure block, if any.
  99  */
 100 void setSuccIRBlocks(IRGS& irgs,
 101                      const RegionDesc& region,
 102                      RegionDesc::BlockId srcBlockId,
 103                      const BlockIdToIRBlockMap& blockIdToIRBlock) {
 104   FTRACE(3, "setSuccIRBlocks: srcBlockId = {}\n", srcBlockId);
 105   auto& irb = *irgs.irb;
 106   irb.resetOffsetMapping();
 107   for (auto dstBlockId : region.succs(srcBlockId)) {
 108     setIRBlock(irgs, dstBlockId, region, blockIdToIRBlock);
 109   }
 110   if (auto nextRetrans = region.nextRetrans(srcBlockId)) {
 111     auto it = blockIdToIRBlock.find(nextRetrans.value());
 112     assertx(it != blockIdToIRBlock.end());
 113     irb.setGuardFailBlock(it->second);
 114   } else {
 115     irb.resetGuardFailBlock();
 116   }
 117 }
 118
 119 bool blockHasUnprocessedPred(
 120   const RegionDesc&             region,
 121   RegionDesc::BlockId           blockId,
 122   const RegionDesc::BlockIdSet& processedBlocks)
 123 {
 124   for (auto predId : region.preds(blockId)) {
 125     if (processedBlocks.count(predId) == 0) {
 126       return true;
 127     }
 128   }
 129   return false;
 130 }
 131
 132 /*
 133  * Returns whether or not block `blockId' is a merge point in the
 134  * `region'.  Normally, blocks are merge points if they have more than
 135  * one predecessor.  However, the region's entry block is a
 136  * merge-point if it has any successor, since it has an implicit arc
 137  * coming from outside of the region.  Additionally, a block with a
 138  * single predecessor is a merge point if it's the target of both the
 139  * fallthru and the taken paths.
 140  */
 141 bool isMerge(const RegionDesc& region, RegionDesc::BlockId blockId) {
 142   auto const& preds = region.preds(blockId);
 143   if (preds.size() == 0)               return false;
 144   if (preds.size() > 1)                return true;
 145   if (blockId == region.entry()->id()) return true;
 146
 147   // The destination of a conditional jump is a merge point if both
 148   // the fallthru and taken offsets are the same.
 149   auto predId   = *preds.begin();
 150   Op* predOpPtr = (Op*)(region.block(predId)->last().pc());
 151   auto predOp   = *predOpPtr;
 152   if (!instrHasConditionalBranch(predOp)) return false;
 153   Offset fallthruOffset = instrLen(predOpPtr);
 154   Offset    takenOffset = *instrJumpOffset(predOpPtr);
 155   return fallthruOffset == takenOffset;
 156 }
 157
 158 /*
 159  * Returns whether any successor of `blockId' in the `region' is a
 160  * merge-point within the `region'.
 161  */
 162 bool hasMergeSucc(const RegionDesc& region,
 163                   RegionDesc::BlockId blockId) {
 164   for (auto succ : region.succs(blockId)) {
 165     if (isMerge(region, succ)) return true;
 166   }
 167   return false;
 168 }
 169
 170
 171 /*
 172  * If this region's entry block is at the entry point for a function, we have
 173  * some additional information we can assume about the types of non-parameter
 174  * local variables.
 175  *
 176  * Note: we can't assume anything if there are DV initializers, because they
 177  * can run arbitrary code before they get to the main entry point (and they do,
 178  * in some hhas-based builtins), if they even go there (they aren't required
 179  * to).
 180  */
 181 void emitEntryAssertions(IRGS& irgs, const Func* func, SrcKey sk) {
 182   if (sk.offset() != func->base()) return;
 183   for (auto& pinfo : func->params()) {
 184     if (pinfo.hasDefaultValue()) return;
 185   }
 186   if (func->isClosureBody()) {
 187     // In a closure, non-parameter locals can have types other than Uninit
 188     // after the prologue runs.  (Local 0 will be the closure itself, and other
 189     // locals will have used vars unpacked into them.)  We rely on hhbbc to
 190     // assert these types.
 191     return;
 192   }
 193   if (func->isPseudoMain()) {
 194     // Pseudomains inherit the variable environment of their caller, so don't
 195     // assert anything in them.
 196     return;
 197   }
 198   auto const numLocs = func->numLocals();
 199   for (auto loc = func->numParams(); loc < numLocs; ++loc) {
 200     auto const location = RegionDesc::Location::Local { loc };
 201     irgen::assertTypeLocation(irgs, location, TUninit);
 202   }
 203 }
 204
 205 /*
 206  * Emit type and reffiness prediction guards.
 207  */
 208 void emitPredictionsAndPreConditions(IRGS& irgs,
 209                           const RegionDesc& region,
 210                           const RegionDesc::BlockPtr block,
 211                           bool isEntry) {
 212   auto const sk = block->start();
 213   auto const bcOff = sk.offset();
 214   auto typePredictions = makeMapWalker(block->typePredictions());
 215   auto typePreConditions = makeMapWalker(block->typePreConditions());
 216   auto refPreds  = makeMapWalker(block->reffinessPreds());
 217
 218   // If the block has a next retranslations in the chain that is a
 219   // merge point in the region, then we need to call
 220   // prepareForHHBCMergePoint to spill the stack.
 221   if (auto retrans = region.nextRetrans(block->id())) {
 222     if (isMerge(region, retrans.value())) {
 223       irgen::prepareForHHBCMergePoint(irgs);
 224     }
 225   }
 226
 227   if (isEntry) {
 228     irgen::ringbufferEntry(irgs, Trace::RBTypeTraceletGuards, sk);
 229     emitEntryAssertions(irgs, block->func(), sk);
 230   }
 231
 232   // Emit type predictions.
 233   while (typePredictions.hasNext(sk)) {
 234     auto const& pred = typePredictions.next();
 235     auto type = pred.type;
 236     auto loc  = pred.location;
 237     irgen::predictTypeLocation(irgs, loc, type);
 238   }
 239
 240   // Emit type guards/preconditions.
 241   while (typePreConditions.hasNext(sk)) {
 242     auto const& preCond = typePreConditions.next();
 243     auto type = preCond.type;
 244     auto loc  = preCond.location;
 245     if (type <= TCls) {
 246       // Do not generate guards for class; instead assert the type.
 247       assertx(loc.tag() == RegionDesc::Location::Tag::Stack);
 248       irgen::assertTypeLocation(irgs, loc, type);
 249     } else {
 250       // Check inner type eagerly if it is the first block during profiling.
 251       // Otherwise only check for BoxedInitCell.
 252       bool checkOuterTypeOnly =
 253         !isEntry || mcg->tx().mode() != TransKind::Profile;
 254       irgen::checkTypeLocation(irgs, loc, type, bcOff, checkOuterTypeOnly);
 255     }
 256   }
 257
 258   // Emit reffiness predictions.
 259   while (refPreds.hasNext(sk)) {
 260     auto const& pred = refPreds.next();
 261     irgen::checkRefs(irgs, pred.arSpOffset, pred.mask, pred.vals, bcOff);
 262   }
 263
 264   // Finish emitting guards, and emit profiling counters.
 265   if (isEntry) {
 266     // With HHIRConstrictGuards, the EndGuards instruction is emitted
 267     // after the guards for the first bytecode instruction.
 268     if (!RuntimeOption::EvalHHIRConstrictGuards) irgen::gen(irgs, EndGuards);
 269
 270     if (RuntimeOption::EvalJitTransCounters) {
 271       irgen::incTransCounter(irgs);
 272     }
 273
 274     if (mcg->tx().mode() == TransKind::Profile) {
 275       if (block->func()->isEntry(bcOff)) {
 276         irgen::checkCold(irgs, mcg->tx().profData()->curTransID());
 277       } else {
 278         irgen::incProfCounter(irgs, mcg->tx().profData()->curTransID());
 279       }
 280     }
 281     irgen::ringbufferEntry(irgs, Trace::RBTypeTraceletBody, sk);
 282
 283     // In the entry block, hhbc-translator gets a chance to emit some code
 284     // immediately after the initial checks on the first instruction.
 285     switch (arch()) {
 286       case Arch::X64:
 287         irgen::prepareEntry(irgs);
 288         break;
 289       case Arch::ARM:
 290         // Don't do this for ARM, because it can lead to interpOne on the
 291         // first SrcKey in a translation, which isn't allowed.
 292         break;
 293     }
 294   }
 295
 296   assertx(!typePredictions.hasNext());
 297   assertx(!refPreds.hasNext());
 298 }
 299
 300 void initNormalizedInstruction(
 301   NormalizedInstruction& inst,
 302   MapWalker<RegionDesc::Block::ParamByRefMap>& byRefs,
 303   IRGS& irgs,
 304   const RegionDesc& region,
 305   RegionDesc::BlockId blockId,
 306   const Func* topFunc,
 307   bool lastInstr,
 308   bool toInterp) {
 309
 310   inst.funcd = topFunc;
 311
 312   if (lastInstr) {
 313     inst.endsRegion  = region.isExit(blockId);
 314     inst.nextIsMerge = hasMergeSucc(region, blockId);
 315   }
 316
 317   // We can get a more precise output type for interpOne if we know all of
 318   // its inputs, so we still populate the rest of the instruction even if
 319   // this is true.
 320   inst.interp = toInterp;
 321
 322   auto const inputInfos = getInputs(inst);
 323
 324   FTRACE(2, "populating inputs for {}\n", inst.toString());
 325   for (auto const& ii : inputInfos) {
 326     inst.inputs.push_back(ii.loc);
 327   }
 328
 329   if (inputInfos.needsRefCheck) {
 330     inst.preppedByRef = byRefs.next();
 331   }
 332 }
 333
 334 bool shouldTrySingletonInline(const RegionDesc& region,
 335                               RegionDesc::BlockPtr block,
 336                               const NormalizedInstruction& inst,
 337                               unsigned instIdx,
 338                               TransFlags trflags) {
 339   if (!RuntimeOption::RepoAuthoritative) return false;
 340
 341   // I don't really want to inline my arm, thanks.
 342   if (arch() != Arch::X64) return false;
 343
 344   // Don't inline if we're retranslating due to a side-exit from an
 345   // inlined call.
 346   auto const startSk = region.start();
 347   if (trflags.noinlineSingleton && startSk == inst.source) return false;
 348
 349   // Bail early if this isn't a push.
 350   if (inst.op() != Op::FPushFuncD &&
 351       inst.op() != Op::FPushClsMethodD) {
 352     return false;
 353   }
 354
 355   return true;
 356 }
 357
 358 /*
 359  * Check if `i' is an FPush{Func,ClsMethod}D followed by an FCall{,D} to a
 360  * function with a singleton pattern, and if so, inline it.  Returns true if
 361  * this succeeds, else false.
 362  */
 363 bool tryTranslateSingletonInline(IRGS& irgs,
 364                                  const NormalizedInstruction& ninst,
 365                                  const Func* funcd) {
 366   using Atom = BCPattern::Atom;
 367   using Captures = BCPattern::CaptureVec;
 368
 369   if (!funcd) return false;
 370
 371   // Make sure we have an acceptable FPush and non-null callee.
 372   assertx(ninst.op() == Op::FPushFuncD ||
 373          ninst.op() == Op::FPushClsMethodD);
 374
 375   auto fcall = ninst.nextSk();
 376
 377   // Check if the next instruction is an acceptable FCall.
 378   if ((fcall.op() != Op::FCall && fcall.op() != Op::FCallD) ||
 379       funcd->isResumable() || funcd->isReturnRef()) {
 380     return false;
 381   }
 382
 383   // First, check for the static local singleton pattern...
 384
 385   // Lambda to check if CGetL and StaticLocInit refer to the same local.
 386   auto has_same_local = [] (PC pc, const Captures& captures) {
 387     if (captures.size() == 0) return false;
 388
 389     auto cgetl = (const Op*)pc;
 390     auto sli = (const Op*)captures[0];
 391
 392     assertx(*cgetl == Op::CGetL);
 393     assertx(*sli == Op::StaticLocInit);
 394
 395     return (getImm(sli, 0).u_IVA == getImm(cgetl, 0).u_IVA);
 396   };
 397
 398   auto cgetl = Atom(Op::CGetL).onlyif(has_same_local);
 399   auto retc  = Atom(Op::RetC);
 400
 401   // Look for a static local singleton pattern.
 402   auto result = BCPattern {
 403     Atom(Op::Null),
 404     Atom(Op::StaticLocInit).capture(),
 405     Atom(Op::IsTypeL),
 406     Atom::alt(
 407       Atom(Op::JmpZ).taken({cgetl, retc}),
 408       Atom::seq(Atom(Op::JmpNZ), cgetl, retc)
 409     )
 410   }.ignore(
 411     {Op::AssertRATL, Op::AssertRATStk}
 412   ).matchAnchored(funcd);
 413
 414   if (result.found()) {
 415     try {
 416       irgen::prepareForNextHHBC(irgs, nullptr, ninst.source, false);
 417       irgen::inlSingletonSLoc(
 418         irgs,
 419         funcd,
 420         (const Op*)result.getCapture(0)
 421       );
 422     } catch (const FailedIRGen& e) {
 423       return false;
 424     } catch (const FailedCodeGen& e) {
 425       return false;
 426     }
 427     TRACE(1, "[singleton-sloc] %s <- %s\n",
 428         funcd->fullName()->data(),
 429         fcall.func()->fullName()->data());
 430     return true;
 431   }
 432
 433   // Not found; check for the static property pattern.
 434
 435   // Factory for String atoms that are required to match another captured
 436   // String opcode.
 437   auto same_string_as = [&] (int i) {
 438     return Atom(Op::String).onlyif([=] (PC pc, const Captures& captures) {
 439       auto string1 = (const Op*)pc;
 440       auto string2 = (const Op*)captures[i];
 441       assertx(*string1 == Op::String);
 442       assertx(*string2 == Op::String);
 443
 444       auto const unit = funcd->unit();
 445       auto sd1 = unit->lookupLitstrId(getImmPtr(string1, 0)->u_SA);
 446       auto sd2 = unit->lookupLitstrId(getImmPtr(string2, 0)->u_SA);
 447
 448       return (sd1 && sd1 == sd2);
 449     });
 450   };
 451
 452   auto stringProp = same_string_as(0);
 453   auto stringCls  = same_string_as(1);
 454   auto agetc = Atom(Op::AGetC);
 455   auto cgets = Atom(Op::CGetS);
 456
 457   // Look for a class static singleton pattern.
 458   result = BCPattern {
 459     Atom(Op::String).capture(),
 460     Atom(Op::String).capture(),
 461     Atom(Op::AGetC),
 462     Atom(Op::CGetS),
 463     Atom(Op::IsTypeC),
 464     Atom::alt(
 465       Atom(Op::JmpZ).taken({stringProp, stringCls, agetc, cgets, retc}),
 466       Atom::seq(Atom(Op::JmpNZ), stringProp, stringCls, agetc, cgets, retc)
 467     )
 468   }.ignore(
 469     {Op::AssertRATL, Op::AssertRATStk}
 470   ).matchAnchored(funcd);
 471
 472   if (result.found()) {
 473     try {
 474       irgen::prepareForNextHHBC(irgs, nullptr, ninst.source, false);
 475       irgen::inlSingletonSProp(
 476         irgs,
 477         funcd,
 478         (const Op*)result.getCapture(1),
 479         (const Op*)result.getCapture(0)
 480       );
 481     } catch (const FailedIRGen& e) {
 482       return false;
 483     } catch (const FailedCodeGen& e) {
 484       return false;
 485     }
 486     TRACE(1, "[singleton-sprop] %s <- %s\n",
 487         funcd->fullName()->data(),
 488         fcall.func()->fullName()->data());
 489     return true;
 490   }
 491
 492   return false;
 493 }
 494
 495 /*
 496  * Returns the id of the next region block in workQ whose
 497  * corresponding IR block is currently reachable from the IR unit's
 498  * entry, or folly::none if no such block exists.  Furthermore, any
 499  * unreachable blocks appearing before the first reachable block are
 500  * moved to the end of workQ.
 501  */
 502 folly::Optional<RegionDesc::BlockId> nextReachableBlock(
 503   jit::queue<RegionDesc::BlockId>& workQ,
 504   const IRBuilder& irb,
 505   const BlockIdToIRBlockMap& blockIdToIRBlock
 506 ) {
 507   auto const size = workQ.size();
 508   for (size_t i = 0; i < size; i++) {
 509     auto const regionBlockId = workQ.front();
 510     workQ.pop();
 511     auto it = blockIdToIRBlock.find(regionBlockId);
 512     assertx(it != blockIdToIRBlock.end());
 513     auto irBlock = it->second;
 514     if (irb.canStartBlock(irBlock)) return regionBlockId;
 515     // Put the block back at the end of workQ, since it may become
 516     // reachable after processing some of the other blocks.
 517     workQ.push(regionBlockId);
 518   }
 519   return folly::none;
 520 }
 521
 522 /*
 523  * Returns whether or not block `bid' is in the retranslation chain
 524  * for `region's entry block.
 525  */
 526 bool inEntryRetransChain(RegionDesc::BlockId bid, const RegionDesc& region) {
 527   auto block = region.entry();
 528   if (block->start() != region.block(bid)->start()) return false;
 529   while (true) {
 530     if (block->id() == bid) return true;
 531     auto nextRetrans = region.nextRetrans(block->id());
 532     if (!nextRetrans) return false;
 533     block = region.block(nextRetrans.value());
 534   }
 535   not_reached();
 536 }
 537
 538 /*
 539  * If `psk' is not an FCall{,D} with inlinable `callee', return nullptr.
 540  *
 541  * Otherwise, select a region for `callee' if one is not already present in
 542  * `retry'.  Update `inl' and return the region if it's inlinable.
 543  */
 544 RegionDescPtr getInlinableCalleeRegion(const ProfSrcKey& psk,
 545                                        const Func* callee,
 546                                        TranslateRetryContext& retry,
 547                                        InliningDecider& inl,
 548                                        const IRGS& irgs) {
 549   if (psk.srcKey.op() != Op::FCall &&
 550       psk.srcKey.op() != Op::FCallD) {
 551     return nullptr;
 552   }
 553
 554   if (!inl.canInlineAt(psk.srcKey, callee)) return nullptr;
 555
 556   auto const& fpiStack = irgs.irb->fpiStack();
 557   // Make sure the FPushOp was in the region
 558   if (fpiStack.empty()) {
 559     return nullptr;
 560   }
 561
 562   // Make sure the FPushOp wasn't interpreted, based on an FPushCuf, or spanned
 563   // another call
 564   auto const info = fpiStack.front();
 565   if (isFPushCuf(info.fpushOpc) || info.interp || info.spansCall) {
 566     return nullptr;
 567   }
 568
 569   // We can't inline FPushClsMethod when the callee may have a $this pointer
 570   if (isFPushClsMethod(info.fpushOpc) && callee->mayHaveThis()) {
 571     return nullptr;
 572   }
 573
 574   RegionDescPtr calleeRegion;
 575   // Look up or select a region for `callee'.
 576   if (retry.inlines.count(psk)) {
 577     calleeRegion = retry.inlines[psk];
 578   } else {
 579     calleeRegion = selectCalleeRegion(psk.srcKey, callee, irgs);
 580     retry.inlines[psk] = calleeRegion;
 581   }
 582   if (!calleeRegion) return nullptr;
 583
 584   // Return the callee region if it's inlinable and update `inl'.
 585   return inl.shouldInline(callee, *calleeRegion) ? calleeRegion
 586                                                  : nullptr;
 587 }
 588
 589 TranslateResult irGenRegion(IRGS& irgs,
 590                             const RegionDesc& region,
 591                             TranslateRetryContext& retry,
 592                             TransFlags trflags,
 593                             InliningDecider& inl) {
 594   const Timer translateRegionTimer(Timer::translateRegion);
 595   FTRACE(1, "translateRegion starting with:\n{}\n", show(region));
 596
 597   if (RuntimeOption::EvalDumpRegion) {
 598     mcg->annotations().emplace_back("RegionDesc", show(region));
 599   }
 600
 601   std::string errorMsg;
 602   always_assert_flog(check(region, errorMsg), "{}", errorMsg);
 603
 604   auto& irb = *irgs.irb;
 605
 606   // Create a map from region blocks to their corresponding initial IR blocks.
 607   auto blockIdToIRBlock = createBlockMap(irgs, region);
 608
 609   if (!inl.inlining()) {
 610     // Prepare to start translation of the first region block.
 611     auto const entry = irb.unit().entry();
 612     irb.startBlock(entry, false /* hasUnprocPred */);
 613
 614     // Make the IR entry block jump to the IR block we mapped the region entry
 615     // block to (they are not the same!).
 616     {
 617       auto const irBlock = blockIdToIRBlock[region.entry()->id()];
 618       always_assert(irBlock != entry);
 619       irgen::gen(irgs, Jmp, irBlock);
 620     }
 621   } else {
 622     // Set the first callee block as a successor to the FCall's block and
 623     // "fallthrough" from the caller into the callee's first block.
 624     setIRBlock(irgs, region.entry()->id(), region, blockIdToIRBlock);
 625     irgen::endBlock(irgs, region.start().offset(), false);
 626   }
 627
 628   RegionDesc::BlockIdSet processedBlocks;
 629
 630   Timer irGenTimer(Timer::translateRegion_irGeneration);
 631   auto& blocks = region.blocks();
 632
 633   jit::queue<RegionDesc::BlockId> workQ;
 634   for (auto& block : blocks) workQ.push(block->id());
 635
 636   while (auto optBlockId = nextReachableBlock(workQ, irb, blockIdToIRBlock)) {
 637     auto const blockId = optBlockId.value();
 638     auto const& block  = region.block(blockId);
 639     auto sk            = block->start();
 640     auto byRefs        = makeMapWalker(block->paramByRefs());
 641     auto knownFuncs    = makeMapWalker(block->knownFuncs());
 642     auto skipTrans     = false;
 643
 644     SCOPE_ASSERT_DETAIL("IRGS") { return show(irgs); };
 645
 646     const Func* topFunc = nullptr;
 647     if (hasTransID(blockId)) irgs.profTransID = getTransID(blockId);
 648     irgs.inlineLevel = inl.depth();
 649     irgs.firstBcInst = inEntryRetransChain(blockId, region) && !inl.inlining();
 650     irgen::prepareForNextHHBC(irgs, nullptr, sk, false);
 651
 652     // Prepare to start translating this region block.  This loads the
 653     // FrameState for the IR block corresponding to the start of this
 654     // region block, and it also sets the map from BC offsets to IR
 655     // blocks for the successors of this block in the region.
 656     auto const irBlock = blockIdToIRBlock[blockId];
 657     const bool hasUnprocPred = blockHasUnprocessedPred(region, blockId,
 658                                                        processedBlocks);
 659     // Note: a block can have an unprocessed predecessor even if the
 660     // region is acyclic, e.g. if the IR was able to prove a path was
 661     // unfeasible due to incompatible types.
 662     if (!irb.startBlock(irBlock, hasUnprocPred)) {
 663       // This can't happen because we picked a reachable block from the workQ.
 664       always_assert_flog(
 665         0, "translateRegion: tried to startBlock on unreachable block {}\n",
 666         blockId
 667       );
 668     }
 669     setSuccIRBlocks(irgs, region, blockId, blockIdToIRBlock);
 670
 671     // Emit an ExitPlaceholder at the beginning of the block if any of
 672     // the optimizations that can benefit from it are enabled, and only
 673     // if we're not inlining. The inlining decision could be smarter
 674     // but this is enough for now since we never emit guards in inlined
 675     // functions (t7385908).
 676     const bool emitExitPlaceholder = irgs.inlineLevel == 0 &&
 677       ((RuntimeOption::EvalHHIRLICM && hasUnprocPred) ||
 678        (RuntimeOption::EvalHHIRTypeCheckHoisting));
 679     if (emitExitPlaceholder) irgen::makeExitPlaceholder(irgs);
 680
 681     // Emit the type and reffiness predictions for this region block. If this is
 682     // the first instruction in the region, we check inner type eagerly, insert
 683     // `EndGuards` after the checks, and generate profiling code in profiling
 684     // translations.
 685     auto const isEntry = block == region.entry() && !inl.inlining();
 686     auto const checkOuterTypeOnly =
 687       !isEntry || mcg->tx().mode() != TransKind::Profile;
 688     emitPredictionsAndPreConditions(irgs, region, block, isEntry);
 689     irb.resetGuardFailBlock();
 690
 691     // Generate IR for each bytecode instruction in this block.
 692     for (unsigned i = 0; i < block->length(); ++i, sk.advance(block->unit())) {
 693       ProfSrcKey psk { irgs.profTransID, sk };
 694       auto const lastInstr = i == block->length() - 1;
 695
 696       // Update bcOff here so any guards or assertions from metadata are
 697       // attributed to this instruction.
 698       irgen::prepareForNextHHBC(irgs, nullptr, sk, false);
 699
 700       // Update the current funcd, if we have a new one.
 701       if (knownFuncs.hasNext(sk)) {
 702         topFunc = knownFuncs.next();
 703       }
 704
 705       // Create and initialize the instruction.
 706       NormalizedInstruction inst(sk, block->unit());
 707       bool toInterpInst = retry.toInterp.count(psk);
 708       initNormalizedInstruction(inst, byRefs, irgs, region, blockId,
 709                                 topFunc, lastInstr, toInterpInst);
 710
 711       // Singleton inlining optimization.
 712       if (RuntimeOption::EvalHHIRInlineSingletons && !lastInstr &&
 713           shouldTrySingletonInline(region, block, inst, i, trflags) &&
 714           knownFuncs.hasNext(inst.nextSk())) {
 715
 716         // This is safe to do even if singleton inlining fails; we just won't
 717         // change topFunc in the next pass since hasNext() will return false.
 718         topFunc = knownFuncs.next();
 719
 720         if (tryTranslateSingletonInline(irgs, inst, topFunc)) {
 721           // Skip the translation of this instruction (the FPush) -and- the
 722           // next instruction (the FCall) if singleton inlining succeeds.
 723           // We still want the fallthrough and prediction logic, though.
 724           skipTrans = true;
 725           continue;
 726         }
 727       }
 728
 729       RegionDescPtr calleeRegion{nullptr};
 730       // See if we have a callee region we can inline---but only if the
 731       // singleton inliner isn't actively inlining.
 732       if (!skipTrans) {
 733         calleeRegion = getInlinableCalleeRegion(psk, inst.funcd, retry, inl,
 734                                                 irgs);
 735       }
 736
 737       if (calleeRegion) {
 738         always_assert(inst.op() == Op::FCall || inst.op() == Op::FCallD);
 739         auto const* callee = inst.funcd;
 740
 741         // We shouldn't be inlining profiling translations.
 742         assert(mcg->tx().mode() != TransKind::Profile);
 743
 744         FTRACE(1, "\nstarting inlined call from {} to {} with {} args "
 745                "and stack:\n{}\n",
 746                block->func()->fullName()->data(),
 747                callee->fullName()->data(),
 748                inst.imm[0].u_IVA,
 749                show(irgs));
 750
 751         auto returnSk = inst.nextSk();
 752         auto returnFuncOff = returnSk.offset() - block->func()->base();
 753
 754         if (irgen::beginInlining(irgs, inst.imm[0].u_IVA, callee,
 755                                  returnFuncOff)) {
 756           SCOPE_ASSERT_DETAIL("Inlined-RegionDesc")
 757             { return show(*calleeRegion); };
 758
 759           // Reset block state before reentering irGenRegion
 760           irb.resetOffsetMapping();
 761           irb.resetGuardFailBlock();
 762
 763           auto result = irGenRegion(irgs, *calleeRegion, retry, trflags, inl);
 764           inl.registerEndInlining(callee);
 765
 766           if (result != TranslateResult::Success) {
 767             // Generating the inlined call failed, bailout
 768             return result;
 769           }
 770
 771           // If this block isn't empty create a new block for the remaining
 772           // instructions
 773           if (!lastInstr) {
 774             auto nextBlock = irb.unit().defBlock();
 775             irb.setBlock(inst.nextSk(), nextBlock);
 776             irgen::endBlock(irgs, inst.nextSk().offset(), inst.nextIsMerge);
 777
 778             // Start a new IR block to hold the remainder of this block.
 779             auto const did_start =
 780               irb.startBlock(nextBlock, false /* unprocessedPred */);
 781             always_assert_flog(did_start,
 782               "Failed to start block following inlined region.");
 783           }
 784
 785           // Recursive calls to irGenRegion will reset the successor block
 786           // mapping
 787           setSuccIRBlocks(irgs, region, blockId, blockIdToIRBlock);
 788
 789           // Don't emit the FCall
 790           skipTrans = true;
 791         }
 792       }
 793
 794       // Emit IR for the body of the instruction.
 795       try {
 796         if (!skipTrans) {
 797           const bool firstInstr = isEntry && i == 0;
 798           translateInstr(irgs, inst, checkOuterTypeOnly, firstInstr);
 799         }
 800       } catch (const FailedIRGen& exn) {
 801         ProfSrcKey psk{irgs.profTransID, sk};
 802         always_assert_flog(!retry.toInterp.count(psk),
 803                            "IR generation failed with {}\n",
 804                            exn.what());
 805         FTRACE(1, "ir generation for {} failed with {}\n",
 806           inst.toString(), exn.what());
 807         retry.toInterp.insert(psk);
 808         return TranslateResult::Retry;
 809       }
 810
 811       irgen::finishHHBC(irgs);
 812
 813       skipTrans = false;
 814
 815       // If this is the last instruction, handle block transitions.
 816       // If the block ends the region, then call irgen::endRegion to
 817       // sync the state and make a REQ_BIND_JMP service request.
 818       // Otherwise, if the instruction has a fall-through, then insert
 819       // a jump to the next offset, since it may not be the next block
 820       // to be translated.
 821       if (lastInstr) {
 822         if (region.isExit(blockId)) {
 823           if (!inl.inlining()) {
 824             irgen::endRegion(irgs);
 825           } else {
 826             assertx(isReturnish(inst.op()));
 827           }
 828         } else if (instrAllowsFallThru(inst.op())) {
 829           if (region.isSideExitingBlock(blockId)) {
 830             irgen::prepareForSideExit(irgs);
 831           }
 832           irgen::endBlock(irgs, inst.nextSk().offset(), inst.nextIsMerge);
 833         }
 834       }
 835     }
 836
 837     processedBlocks.insert(blockId);
 838
 839     assertx(!byRefs.hasNext());
 840     assertx(!knownFuncs.hasNext());
 841   }
 842
 843   if (!inl.inlining()) {
 844     irgen::sealUnit(irgs);
 845   } else {
 846     always_assert_flog(irgs.inlineLevel == inl.depth() - 1,
 847                        "Tried to inline a region with no return.");
 848   }
 849
 850   irGenTimer.stop();
 851   return TranslateResult::Success;
 852 }
 853
 854 TranslateResult mcGenRegion(IRGS& irgs,
 855                             const RegionDesc& region,
 856                             ProfSrcKeySet& toInterp) {
 857   auto const startSk = region.start();
 858   try {
 859     mcg->traceCodeGen(irgs);
 860     if (mcg->tx().mode() == TransKind::Profile) {
 861       mcg->tx().profData()->setProfiling(startSk.func()->getFuncId());
 862     }
 863   } catch (const FailedCodeGen& exn) {
 864     SrcKey sk{exn.vmFunc, exn.bcOff, exn.resumed};
 865     ProfSrcKey psk{exn.profTransId, sk};
 866     always_assert_log(
 867       !toInterp.count(psk),
 868       [&] {
 869         std::ostringstream oss;
 870         oss << folly::format("code generation failed with {}\n", exn.what());
 871         print(oss, irgs.irb->unit());
 872         return oss.str();
 873       });
 874     toInterp.insert(psk);
 875     return TranslateResult::Retry;
 876   } catch (const DataBlockFull& dbFull) {
 877     if (dbFull.name == "hot") {
 878       assertx(mcg->tx().useAHot());
 879       mcg->tx().setUseAHot(false);
 880       // We can't return Retry here because the code block selection
 881       // will still say hot.
 882       return TranslateResult::Failure;
 883     } else {
 884       always_assert_flog(0, "data block = {}\nmessage: {}\n",
 885                          dbFull.name, dbFull.what());
 886     }
 887   }
 888   return TranslateResult::Success;
 889 }
 890
 891 }
 892
 893 //////////////////////////////////////////////////////////////////////
 894
 895 TranslateResult translateRegion(IRGS& irgs,
 896                                 const RegionDesc& region,
 897                                 TranslateRetryContext& retry,
 898                                 TransFlags trflags,
 899                                 PostConditions& pConds) {
 900   SCOPE_ASSERT_DETAIL("RegionDesc") { return show(region); };
 901   SCOPE_ASSERT_DETAIL("IRUnit") { return show(irgs.unit); };
 902
 903   // Set up inlining context, but disable it for profiling mode.
 904   InliningDecider inl(region.entry()->func());
 905   if (mcg->tx().mode() == TransKind::Profile) inl.disable();
 906
 907   auto irGenResult = irGenRegion(irgs, region, retry, trflags, inl);
 908   if (irGenResult != TranslateResult::Success) return irGenResult;
 909
 910   // For profiling translations, grab the postconditions to be used
 911   // for region selection whenever we decide to retranslate.
 912   pConds.changed.clear();
 913   pConds.refined.clear();
 914   if (mcg->tx().mode() == TransKind::Profile &&
 915       RuntimeOption::EvalJitPGOUsePostConditions) {
 916     auto& unit = irgs.irb->unit();
 917     auto  lastSrcKey = region.lastSrcKey();
 918     Block* mainExit = findMainExitBlock(unit, lastSrcKey);
 919     FTRACE(2, "translateRegion: mainExit: B{}\nUnit: {}\n",
 920            mainExit->id(), show(unit));
 921     assertx(mainExit);
 922     pConds = irgs.irb->postConds(mainExit);
 923   }
 924
 925   return mcGenRegion(irgs, region, retry.toInterp);
 926 }
 927
 928 } }