js/src/jit/TrialInlining.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/TrialInlining.h"
   8
   9 #include "jit/BaselineCacheIRCompiler.h"
  10 #include "jit/BaselineIC.h"
  11 #include "jit/Ion.h"  // TooManyFormalArguments
  12
  13 #include "jit/BaselineFrame-inl.h"
  14 #include "vm/BytecodeIterator-inl.h"
  15 #include "vm/BytecodeLocation-inl.h"
  16
  17 using mozilla::Maybe;
  18
  19 namespace js {
  20 namespace jit {
  21
  22 bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) {
  23   MOZ_ASSERT(JitOptions.warpBuilder);
  24
  25   RootedScript script(cx, frame->script());
  26   ICScript* icScript = frame->icScript();
  27   bool isRecursive = !!icScript->inliningRoot();
  28
  29   if (!script->canIonCompile()) {
  30     return true;
  31   }
  32
  33   const uint32_t MAX_INLINING_DEPTH = 5;
  34   if (icScript->depth() > MAX_INLINING_DEPTH) {
  35     return true;
  36   }
  37
  38   InliningRoot* root =
  39       isRecursive ? icScript->inliningRoot()
  40                   : script->jitScript()->getOrCreateInliningRoot(cx, script);
  41   if (!root) {
  42     return false;
  43   }
  44
  45   JitSpew(JitSpew_WarpTrialInlining,
  46           "Trial inlining for %s script %s:%u:%u (%p) (inliningRoot=%p)",
  47           (isRecursive ? "inner" : "outer"), script->filename(),
  48           script->lineno(), script->column(), frame->script(), root);
  49
  50   TrialInliner inliner(cx, script, icScript, root);
  51   return inliner.tryInlining();
  52 }
  53
  54 void TrialInliner::cloneSharedPrefix(ICStub* stub, const uint8_t* endOfPrefix,
  55                                      CacheIRWriter& writer) {
  56   CacheIRReader reader(stub->cacheIRStubInfo());
  57   CacheIRCloner cloner(stub);
  58   while (reader.currentPosition() < endOfPrefix) {
  59     CacheOp op = reader.readOp();
  60     cloner.cloneOp(op, reader, writer);
  61   }
  62 }
  63
  64 void TrialInliner::replaceICStub(const ICEntry& entry, CacheIRWriter& writer,
  65                                  CacheKind kind) {
  66   ICFallbackStub* fallbackStub = entry.fallbackStub();
  67   fallbackStub->discardStubs(cx(), root_->owningScript());
  68
  69   // Note: AttachBaselineCacheIRStub never throws an exception.
  70   bool attached = false;
  71   ICStub* newStub = AttachBaselineCacheIRStub(
  72       cx(), writer, kind, BaselineCacheIRStubKind::Regular, script_, icScript_,
  73       fallbackStub, &attached);
  74   if (newStub) {
  75     MOZ_ASSERT(attached);
  76     JitSpew(JitSpew_WarpTrialInlining, "Attached new stub %p", newStub);
  77   }
  78 }
  79
  80 ICStub* TrialInliner::maybeSingleStub(const ICEntry& entry) {
  81   // Look for a single non-fallback stub followed by stubs with entered-count 0.
  82   // Allow one optimized stub before the fallback stub to support the
  83   // CallIRGenerator::emitCalleeGuard optimization where we first try a
  84   // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
  85   ICStub* stub = entry.firstStub();
  86   if (stub->isFallback()) {
  87     return nullptr;
  88   }
  89   ICStub* next = stub->next();
  90   if (next->getEnteredCount() != 0) {
  91     return nullptr;
  92   }
  93   if (!next->isFallback()) {
  94     ICStub* nextNext = next->next();
  95     if (!nextNext->isFallback() || nextNext->getEnteredCount() != 0) {
  96       return nullptr;
  97     }
  98   }
  99   return stub;
 100 }
 101
 102 Maybe<InlinableCallData> FindInlinableCallData(ICStub* stub) {
 103   Maybe<InlinableCallData> data;
 104
 105   const CacheIRStubInfo* stubInfo = stub->cacheIRStubInfo();
 106   const uint8_t* stubData = stub->cacheIRStubData();
 107
 108   ObjOperandId calleeGuardOperand;
 109   CallFlags flags;
 110   JSFunction* target = nullptr;
 111
 112   CacheIRReader reader(stubInfo);
 113   while (reader.more()) {
 114     const uint8_t* opStart = reader.currentPosition();
 115
 116     CacheOp op = reader.readOp();
 117     uint32_t argLength = CacheIROpArgLengths[size_t(op)];
 118     mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
 119
 120     switch (op) {
 121       case CacheOp::GuardSpecificFunction: {
 122         // If we see a guard, remember which operand we are guarding.
 123         MOZ_ASSERT(data.isNothing());
 124         calleeGuardOperand = reader.objOperandId();
 125         uint32_t targetOffset = reader.stubOffset();
 126         mozilla::Unused << reader.stubOffset();  // nargsAndFlags
 127         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
 128         target = reinterpret_cast<JSFunction*>(rawTarget);
 129         break;
 130       }
 131       case CacheOp::GuardFunctionScript: {
 132         MOZ_ASSERT(data.isNothing());
 133         calleeGuardOperand = reader.objOperandId();
 134         uint32_t targetOffset = reader.stubOffset();
 135         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
 136         target = reinterpret_cast<BaseScript*>(rawTarget)->function();
 137         mozilla::Unused << reader.stubOffset();  // nargsAndFlags
 138         break;
 139       }
 140       case CacheOp::CallScriptedFunction: {
 141         // If we see a call, check if `callee` is the previously guarded
 142         // operand. If it is, we know the target and can inline.
 143         ObjOperandId calleeOperand = reader.objOperandId();
 144         mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
 145         flags = reader.callFlags();
 146
 147         if (calleeOperand == calleeGuardOperand) {
 148           MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
 149           MOZ_ASSERT(data.isNothing());
 150           data.emplace();
 151           data->endOfSharedPrefix = opStart;
 152         }
 153         break;
 154       }
 155       case CacheOp::CallInlinedFunction: {
 156         ObjOperandId calleeOperand = reader.objOperandId();
 157         mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
 158         uint32_t icScriptOffset = reader.stubOffset();
 159         flags = reader.callFlags();
 160
 161         if (calleeOperand == calleeGuardOperand) {
 162           MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
 163           MOZ_ASSERT(data.isNothing());
 164           data.emplace();
 165           data->endOfSharedPrefix = opStart;
 166           uintptr_t rawICScript =
 167               stubInfo->getStubRawWord(stubData, icScriptOffset);
 168           data->icScript = reinterpret_cast<ICScript*>(rawICScript);
 169         }
 170         break;
 171       }
 172       default:
 173         if (data.isSome()) {
 174           MOZ_ASSERT(op == CacheOp::ReturnFromIC ||
 175                      op == CacheOp::TypeMonitorResult);
 176         }
 177         reader.skip(argLength);
 178         break;
 179     }
 180     MOZ_ASSERT(argStart + argLength == reader.currentPosition());
 181   }
 182
 183   if (data.isSome()) {
 184     data->calleeOperand = calleeGuardOperand;
 185     data->callFlags = flags;
 186     data->target = target;
 187   }
 188   return data;
 189 }
 190
 191 /*static*/
 192 bool TrialInliner::canInline(JSFunction* target, HandleScript caller) {
 193   if (!target->hasJitScript()) {
 194     return false;
 195   }
 196   JSScript* script = target->nonLazyScript();
 197   if (!script->jitScript()->hasBaselineScript() || script->uninlineable() ||
 198       !script->canIonCompile() || script->needsArgsObj() ||
 199       script->isDebuggee()) {
 200     return false;
 201   }
 202   // Don't inline cross-realm calls.
 203   if (target->realm() != caller->realm()) {
 204     return false;
 205   }
 206   return true;
 207 }
 208
 209 bool TrialInliner::shouldInline(JSFunction* target, ICStub* stub,
 210                                 BytecodeLocation loc) {
 211   if (!canInline(target, script_)) {
 212     return false;
 213   }
 214   JitSpew(JitSpew_WarpTrialInlining,
 215           "Inlining candidate JSOp::%s: callee script %s:%u:%u",
 216           CodeName(loc.getOp()), target->nonLazyScript()->filename(),
 217           target->nonLazyScript()->lineno(), target->nonLazyScript()->column());
 218
 219   uint32_t entryCount = stub->getEnteredCount();
 220   if (entryCount < JitOptions.inliningEntryThreshold) {
 221     JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)",
 222             unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold));
 223     return false;
 224   }
 225
 226   if (!JitOptions.isSmallFunction(target->nonLazyScript())) {
 227     JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)",
 228             unsigned(target->nonLazyScript()->length()),
 229             unsigned(JitOptions.smallFunctionMaxBytecodeLength));
 230     return false;
 231   }
 232
 233   if (TooManyFormalArguments(target->nargs())) {
 234     JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u",
 235             unsigned(target->nargs()));
 236     return false;
 237   }
 238
 239   if (TooManyFormalArguments(loc.getCallArgc())) {
 240     JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u",
 241             unsigned(loc.getCallArgc()));
 242     return false;
 243   }
 244
 245   return true;
 246 }
 247
 248 ICScript* TrialInliner::createInlinedICScript(JSFunction* target,
 249                                               BytecodeLocation loc) {
 250   MOZ_ASSERT(target->hasJitEntry());
 251   MOZ_ASSERT(target->hasJitScript());
 252
 253   JSScript* targetScript = target->baseScript()->asJSScript();
 254
 255   // We don't have to check for overflow here because we have already
 256   // successfully allocated an ICScript with this number of entries
 257   // when creating the JitScript for the target function, and we
 258   // checked for overflow then.
 259   uint32_t allocSize =
 260       sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry);
 261
 262   void* raw = cx()->pod_malloc<uint8_t>(allocSize);
 263   MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0);
 264   if (!raw) {
 265     return nullptr;
 266   }
 267
 268   // TODO: Increase this to make recursive inlining more aggressive.
 269   // Alternatively, we could add a trialInliningThreshold field to
 270   // ICScript to give more precise control.
 271   const uint32_t InitialWarmUpCount = 0;
 272
 273   uint32_t depth = icScript_->depth() + 1;
 274   UniquePtr<ICScript> inlinedICScript(
 275       new (raw) ICScript(InitialWarmUpCount, allocSize, depth, root_));
 276
 277   {
 278     // Suppress GC. This matches the AutoEnterAnalysis in
 279     // JSScript::createJitScript. It is needed for allocating the
 280     // template object for JSOp::Rest and the object group for
 281     // JSOp::NewArray.
 282     gc::AutoSuppressGC suppress(cx());
 283     if (!inlinedICScript->initICEntries(cx(), targetScript)) {
 284       return nullptr;
 285     }
 286   }
 287
 288   uint32_t pcOffset = loc.bytecodeToOffset(script_);
 289   ICScript* result = inlinedICScript.get();
 290   if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) {
 291     return nullptr;
 292   }
 293   MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries());
 294
 295   JitSpew(JitSpew_WarpTrialInlining,
 296           "Outer ICScript: %p Inner ICScript: %p pcOffset: %u\n", icScript_,
 297           result, pcOffset);
 298
 299   return result;
 300 }
 301
 302 bool TrialInliner::maybeInlineCall(const ICEntry& entry, BytecodeLocation loc) {
 303   ICStub* stub = maybeSingleStub(entry);
 304   if (!stub) {
 305     return true;
 306   }
 307
 308   // Ensure that we haven't already trial-inlined a different stub.
 309   uint32_t pcOffset = loc.bytecodeToOffset(script_);
 310   if (!stub->next()->isFallback()) {
 311     if (icScript_->hasInlinedChild(pcOffset)) {
 312       return true;
 313     }
 314   }
 315
 316   // Look for a CallScriptedFunction with a known target.
 317   Maybe<InlinableCallData> data = FindInlinableCallData(stub);
 318   if (data.isNothing()) {
 319     return true;
 320   }
 321   // Ensure that we haven't already trial-inlined this stub.
 322   if (data->icScript) {
 323     return true;
 324   }
 325
 326   MOZ_ASSERT(!icScript_->hasInlinedChild(pcOffset));
 327
 328   // Decide whether to inline the target.
 329   if (!shouldInline(data->target, stub, loc)) {
 330     return true;
 331   }
 332
 333   // We only inline FunCall if we are calling the js::fun_call builtin.
 334   MOZ_ASSERT_IF(loc.getOp() == JSOp::FunCall,
 335                 data->callFlags.getArgFormat() == CallFlags::FunCall);
 336
 337   ICScript* newICScript = createInlinedICScript(data->target, loc);
 338   if (!newICScript) {
 339     return false;
 340   }
 341
 342   CacheIRWriter writer(cx());
 343   Int32OperandId argcId(writer.setInputOperandId(0));
 344   cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
 345
 346   writer.callInlinedFunction(data->calleeOperand, argcId, newICScript,
 347                              data->callFlags);
 348   writer.returnFromIC();
 349
 350   replaceICStub(entry, writer, CacheKind::Call);
 351   return true;
 352 }
 353
 354 bool TrialInliner::tryInlining() {
 355   uint32_t icIndex = 0;
 356   for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
 357     JSOp op = loc.getOp();
 358     switch (op) {
 359       case JSOp::Call:
 360       case JSOp::CallIgnoresRv:
 361       case JSOp::CallIter:
 362       case JSOp::FunCall:
 363         if (!maybeInlineCall(icScript_->icEntry(icIndex), loc)) {
 364           return false;
 365         }
 366         break;
 367       default:
 368         break;
 369     }
 370     if (loc.opHasIC()) {
 371       icIndex++;
 372     }
 373   }
 374
 375   return true;
 376 }
 377
 378 bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) {
 379   return inlinedScripts_.append(std::move(icScript));
 380 }
 381
 382 void InliningRoot::removeInlinedScript(ICScript* icScript) {
 383   inlinedScripts_.eraseIf(
 384       [icScript](const UniquePtr<ICScript>& script) -> bool {
 385         return script.get() == icScript;
 386       });
 387 }
 388
 389 void InliningRoot::trace(JSTracer* trc) {
 390   TraceEdge(trc, &owningScript_, "inlining-root-owning-script");
 391   for (auto& inlinedScript : inlinedScripts_) {
 392     inlinedScript->trace(trc);
 393   }
 394 }
 395
 396 void InliningRoot::purgeOptimizedStubs(Zone* zone) {
 397   for (auto& inlinedScript : inlinedScripts_) {
 398     inlinedScript->purgeOptimizedStubs(zone);
 399   }
 400 }
 401
 402 }  // namespace jit
 403 }  // namespace js