From d0316244345f3d319c13439b2970019b83f2895a Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Wed, 7 Feb 2018 09:27:23 -0800 Subject: [PATCH] Tweak inlining adjustment factor Summary: The current heuristic just looks at the hotness of the caller, relative to the average hotness of all function entries. The trouble with this is that there could be 50 callers that are all significantly hotter than average; but in that case its almost certainly better not to inline big functions. To try to avoid overweighting, the current heuristic adds a constant factor - defaulting to 0.1 - so that functions have to be at least 10x hotter than average before they get higher weighting; this probably mitigates the lage-number-of-callers issue in most cases. This changes the adjustment factor to pow(callerWight/averageWeight, callerExp) * pow(callerWeight/calleeWeight, calleeExp) where callerExp and calleeExp are runtime options defaulting to 0.5 This means that callees with few callers get higher weighting than callees with many callers. Reviewed By: ottoni Differential Revision: D6832577 fbshipit-source-id: 18b0524edbe8401cf4b68af29a51734fa001f489 --- hphp/runtime/base/runtime-option.h | 3 ++- hphp/runtime/vm/jit/inlining-decider.cpp | 30 ++++++++++++++++++++---------- hphp/runtime/vm/jit/irgen.cpp | 10 ++++++++-- hphp/runtime/vm/jit/irgen.h | 4 +++- hphp/runtime/vm/jit/translate-region.h | 3 ++- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/hphp/runtime/base/runtime-option.h b/hphp/runtime/base/runtime-option.h index b256f34d025..e95c8029738 100644 --- a/hphp/runtime/base/runtime-option.h +++ b/hphp/runtime/base/runtime-option.h @@ -616,7 +616,8 @@ struct RuntimeOption { F(uint32_t, HHIRInliningVasmCostLimit, 175) \ F(uint32_t, HHIRInliningMinVasmCostLimit, 100) \ F(uint32_t, HHIRInliningMaxVasmCostLimit, 400) \ - F(double, HHIRInliningVasmCostFactor, 0.1) \ + F(double, HHIRInliningVasmCallerExp, .5) \ + F(double, HHIRInliningVasmCalleeExp, .5) \ F(uint32_t, HHIRInliningMaxReturnDecRefs, 12) \ F(uint32_t, HHIRInliningMaxReturnLocals, 20) \ F(bool, HHIRInlineFrameOpts, true) \ diff --git a/hphp/runtime/vm/jit/inlining-decider.cpp b/hphp/runtime/vm/jit/inlining-decider.cpp index 42810e33be7..910a642cb5d 100644 --- a/hphp/runtime/vm/jit/inlining-decider.cpp +++ b/hphp/runtime/vm/jit/inlining-decider.cpp @@ -39,6 +39,7 @@ #include #include +#include #include namespace HPHP { namespace jit { @@ -375,12 +376,20 @@ int computeTranslationCost(SrcKey at, Op callerFPushOp, return cost; } -uint64_t adjustedMaxVasmCost(uint64_t curProfCount) { +uint64_t adjustedMaxVasmCost(const irgen::IRGS& env, + const RegionDesc& calleeRegion) { const auto baseVasmCost = RuntimeOption::EvalHHIRInliningVasmCostLimit; - if (s_baseProfCount.load() == 0) return baseVasmCost; const auto baseProfCount = s_baseProfCount.load(); - const auto costFactor = RuntimeOption::EvalHHIRInliningVasmCostFactor; - auto adjustedCost = costFactor * baseVasmCost * curProfCount / baseProfCount; + if (baseProfCount == 0) return baseVasmCost; + auto const callerProfCount = irgen::curProfCount(env); + auto adjustedCost = baseVasmCost * + std::pow((double)callerProfCount / baseProfCount, + RuntimeOption::EvalHHIRInliningVasmCallerExp); + auto const calleeProfCount = irgen::calleeProfCount(env, calleeRegion); + if (calleeProfCount) { + adjustedCost *= std::pow((double)callerProfCount / calleeProfCount, + RuntimeOption::EvalHHIRInliningVasmCalleeExp); + } if (adjustedCost < RuntimeOption::EvalHHIRInliningMinVasmCostLimit) { adjustedCost = RuntimeOption::EvalHHIRInliningMinVasmCostLimit; } @@ -388,8 +397,8 @@ uint64_t adjustedMaxVasmCost(uint64_t curProfCount) { adjustedCost = RuntimeOption::EvalHHIRInliningMaxVasmCostLimit; } FTRACE(3, "adjustedMaxVasmCost: adjustedCost ({}) = baseVasmCost ({}) * " - "curProfCount ({}) / baseProfCount ({})\n", - adjustedCost, baseVasmCost, curProfCount, baseProfCount); + "callerProfCount ({}) / baseProfCount ({})\n", + adjustedCost, baseVasmCost, callerProfCount, baseProfCount); return adjustedCost; } @@ -404,7 +413,7 @@ int InliningDecider::accountForInlining(SrcKey callerSk, const Func* callee, const RegionDesc& region, const irgen::IRGS& irgs) { - const auto maxCost = adjustedMaxVasmCost(curProfCount(irgs)); + const auto maxCost = adjustedMaxVasmCost(irgs, region); int cost = computeTranslationCost(callerSk, callerFPushOp, region, maxCost); m_costStack.push_back(cost); m_cost += cost; @@ -694,13 +703,13 @@ RegionDescPtr selectCalleeRegion(const SrcKey& sk, const auto mode = RuntimeOption::EvalInlineRegionMode; - const auto adjustedMaxCost = adjustedMaxVasmCost(curProfCount(irgs)); if (mode == "cfg" || mode == "both") { if (profData()) { auto region = selectCalleeCFG(callee, numArgs, ctx, argTypes, maxBCInstrs); if (region && inl.shouldInline(sk, fpiInfo.fpushOpc, - callee, *region, adjustedMaxCost)) { + callee, *region, + adjustedMaxVasmCost(irgs, *region))) { return region; } } @@ -717,7 +726,8 @@ RegionDescPtr selectCalleeRegion(const SrcKey& sk, ); if (region && inl.shouldInline(sk, fpiInfo.fpushOpc, - callee, *region, adjustedMaxCost)) { + callee, *region, + adjustedMaxVasmCost(irgs, *region))) { return region; } diff --git a/hphp/runtime/vm/jit/irgen.cpp b/hphp/runtime/vm/jit/irgen.cpp index 9623323137e..9c3030d8fa8 100644 --- a/hphp/runtime/vm/jit/irgen.cpp +++ b/hphp/runtime/vm/jit/irgen.cpp @@ -56,11 +56,17 @@ void check_catch_stack_state(IRGS& env, const IRInstruction* inst) { } uint64_t curProfCount(const IRGS& env) { - auto tid = env.profTransID; + auto const tid = env.profTransID; assertx(tid == kInvalidTransID || (env.region != nullptr && profData() != nullptr)); return env.profFactor * - (tid != kInvalidTransID ? env.region->blockProfCount(tid) : 1); + (tid != kInvalidTransID ? env.region->blockProfCount(tid) : 1); +} + +uint64_t calleeProfCount(const IRGS& env, const RegionDesc& calleeRegion) { + auto const tid = calleeRegion.entry()->id(); + if (tid == kInvalidTransID) return 0; + return env.profFactor * calleeRegion.blockProfCount(tid); } ////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/vm/jit/irgen.h b/hphp/runtime/vm/jit/irgen.h index 7d0bba82187..f9f7fb777d7 100644 --- a/hphp/runtime/vm/jit/irgen.h +++ b/hphp/runtime/vm/jit/irgen.h @@ -49,13 +49,14 @@ struct StringData; namespace jit { struct Block; -struct IRGS; struct IRInstruction; struct NormalizedInstruction; struct SSATmp; namespace irgen { +struct IRGS; + /////////////////////////////////////////////////////////////////////////////// /* @@ -152,6 +153,7 @@ void incProfCounter(IRGS&, TransID); void checkCold(IRGS&, TransID); uint64_t curProfCount(const IRGS& env); +uint64_t calleeProfCount(const IRGS& env, const RegionDesc& calleeRegion); /* * If ringbuffer tracing is enabled, generate a ringbuffer entry associated diff --git a/hphp/runtime/vm/jit/translate-region.h b/hphp/runtime/vm/jit/translate-region.h index a3adfe2a199..7444f5c0c6f 100644 --- a/hphp/runtime/vm/jit/translate-region.h +++ b/hphp/runtime/vm/jit/translate-region.h @@ -23,7 +23,8 @@ namespace HPHP { namespace jit { -struct IRGS; +namespace irgen { struct IRGS; } + struct IRUnit; struct TransContext; -- 2.11.4.GIT