js/src/jit/TrialInlining.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/TrialInlining.h"
   8
   9 #include "mozilla/DebugOnly.h"
  10
  11 #include "jit/BaselineCacheIRCompiler.h"
  12 #include "jit/BaselineFrame.h"
  13 #include "jit/BaselineIC.h"
  14 #include "jit/CacheIRCloner.h"
  15 #include "jit/CacheIRHealth.h"
  16 #include "jit/CacheIRWriter.h"
  17 #include "jit/Ion.h"  // TooManyFormalArguments
  18
  19 #include "vm/BytecodeLocation-inl.h"
  20
  21 using mozilla::Maybe;
  22
  23 namespace js {
  24 namespace jit {
  25
  26 bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) {
  27   RootedScript script(cx, frame->script());
  28   ICScript* icScript = frame->icScript();
  29   bool isRecursive = icScript->depth() > 0;
  30
  31 #ifdef JS_CACHEIR_SPEW
  32   if (cx->spewer().enabled(cx, script, SpewChannel::CacheIRHealthReport)) {
  33     for (uint32_t i = 0; i < icScript->numICEntries(); i++) {
  34       ICEntry& entry = icScript->icEntry(i);
  35       ICFallbackStub* fallbackStub = icScript->fallbackStub(i);
  36
  37       // If the IC is megamorphic or generic, then we have already
  38       // spewed the IC report on transition.
  39       if (!(uint8_t(fallbackStub->state().mode()) > 0)) {
  40         jit::ICStub* stub = entry.firstStub();
  41         bool sawNonZeroCount = false;
  42         while (!stub->isFallback()) {
  43           uint32_t count = stub->enteredCount();
  44           if (count > 0 && sawNonZeroCount) {
  45             CacheIRHealth cih;
  46             cih.healthReportForIC(cx, &entry, fallbackStub, script,
  47                                   SpewContext::TrialInlining);
  48             break;
  49           }
  50
  51           if (count > 0 && !sawNonZeroCount) {
  52             sawNonZeroCount = true;
  53           }
  54
  55           stub = stub->toCacheIRStub()->next();
  56         }
  57       }
  58     }
  59   }
  60 #endif
  61
  62   if (!script->canIonCompile()) {
  63     return true;
  64   }
  65
  66   // Baseline shouldn't attempt trial inlining in scripts that are too large.
  67   MOZ_ASSERT_IF(JitOptions.limitScriptSize,
  68                 script->length() <= JitOptions.ionMaxScriptSize);
  69
  70   const uint32_t MAX_INLINING_DEPTH = 4;
  71   if (icScript->depth() > MAX_INLINING_DEPTH) {
  72     return true;
  73   }
  74
  75   InliningRoot* root = isRecursive ? icScript->inliningRoot()
  76                                    : script->jitScript()->inliningRoot();
  77   if (JitSpewEnabled(JitSpew_WarpTrialInlining)) {
  78     // Eagerly create the inlining root when it's used in the spew output.
  79     if (!root) {
  80       MOZ_ASSERT(!isRecursive);
  81       root = script->jitScript()->getOrCreateInliningRoot(cx, script);
  82       if (!root) {
  83         return false;
  84       }
  85     }
  86     UniqueChars funName;
  87     if (script->function() && script->function()->fullDisplayAtom()) {
  88       funName =
  89           AtomToPrintableString(cx, script->function()->fullDisplayAtom());
  90     }
  91
  92     JitSpew(
  93         JitSpew_WarpTrialInlining,
  94         "Trial inlining for %s script '%s' (%s:%u:%u (%p)) (inliningRoot=%p)",
  95         (isRecursive ? "inner" : "outer"),
  96         funName ? funName.get() : "<unnamed>", script->filename(),
  97         script->lineno(), script->column().oneOriginValue(), frame->script(),
  98         root);
  99     JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
 100   }
 101
 102   TrialInliner inliner(cx, script, icScript);
 103   return inliner.tryInlining();
 104 }
 105
 106 void TrialInliner::cloneSharedPrefix(ICCacheIRStub* stub,
 107                                      const uint8_t* endOfPrefix,
 108                                      CacheIRWriter& writer) {
 109   CacheIRReader reader(stub->stubInfo());
 110   CacheIRCloner cloner(stub);
 111   while (reader.currentPosition() < endOfPrefix) {
 112     CacheOp op = reader.readOp();
 113     cloner.cloneOp(op, reader, writer);
 114   }
 115 }
 116
 117 bool TrialInliner::replaceICStub(ICEntry& entry, ICFallbackStub* fallback,
 118                                  CacheIRWriter& writer, CacheKind kind) {
 119   MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
 120
 121   fallback->discardStubs(cx()->zone(), &entry);
 122
 123   // Note: AttachBaselineCacheIRStub never throws an exception.
 124   ICAttachResult result = AttachBaselineCacheIRStub(
 125       cx(), writer, kind, script_, icScript_, fallback, "TrialInline");
 126   if (result == ICAttachResult::Attached) {
 127     MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Inlined);
 128     return true;
 129   }
 130
 131   MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
 132   icScript_->removeInlinedChild(fallback->pcOffset());
 133
 134   if (result == ICAttachResult::OOM) {
 135     ReportOutOfMemory(cx());
 136     return false;
 137   }
 138
 139   // We failed to attach a new IC stub due to CacheIR size limits. Disable trial
 140   // inlining for this location and return true.
 141   MOZ_ASSERT(result == ICAttachResult::TooLarge);
 142   fallback->setTrialInliningState(TrialInliningState::Failure);
 143   return true;
 144 }
 145
 146 ICCacheIRStub* TrialInliner::maybeSingleStub(const ICEntry& entry) {
 147   // Look for a single non-fallback stub followed by stubs with entered-count 0.
 148   // Allow one optimized stub before the fallback stub to support the
 149   // CallIRGenerator::emitCalleeGuard optimization where we first try a
 150   // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
 151   ICStub* stub = entry.firstStub();
 152   if (stub->isFallback()) {
 153     return nullptr;
 154   }
 155   ICStub* next = stub->toCacheIRStub()->next();
 156   if (next->enteredCount() != 0) {
 157     return nullptr;
 158   }
 159
 160   ICFallbackStub* fallback = nullptr;
 161   if (next->isFallback()) {
 162     fallback = next->toFallbackStub();
 163   } else {
 164     ICStub* nextNext = next->toCacheIRStub()->next();
 165     if (!nextNext->isFallback() || nextNext->enteredCount() != 0) {
 166       return nullptr;
 167     }
 168     fallback = nextNext->toFallbackStub();
 169   }
 170
 171   if (fallback->trialInliningState() != TrialInliningState::Candidate) {
 172     return nullptr;
 173   }
 174
 175   return stub->toCacheIRStub();
 176 }
 177
 178 Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub,
 179                                            BytecodeLocation loc) {
 180   if (loc.isInvokeOp()) {
 181     Maybe<InlinableCallData> call = FindInlinableCallData(stub);
 182     if (call.isSome()) {
 183       return call;
 184     }
 185   }
 186   if (loc.isGetPropOp() || loc.isGetElemOp()) {
 187     Maybe<InlinableGetterData> getter = FindInlinableGetterData(stub);
 188     if (getter.isSome()) {
 189       return getter;
 190     }
 191   }
 192   if (loc.isSetPropOp()) {
 193     Maybe<InlinableSetterData> setter = FindInlinableSetterData(stub);
 194     if (setter.isSome()) {
 195       return setter;
 196     }
 197   }
 198   return mozilla::Nothing();
 199 }
 200
 201 Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub) {
 202   Maybe<InlinableCallData> data;
 203
 204   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 205   const uint8_t* stubData = stub->stubDataStart();
 206
 207   ObjOperandId calleeGuardOperand;
 208   CallFlags flags;
 209   JSFunction* target = nullptr;
 210
 211   CacheIRReader reader(stubInfo);
 212   while (reader.more()) {
 213     const uint8_t* opStart = reader.currentPosition();
 214
 215     CacheOp op = reader.readOp();
 216     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 217     uint32_t argLength = opInfo.argLength;
 218     mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
 219
 220     switch (op) {
 221       case CacheOp::GuardSpecificFunction: {
 222         // If we see a guard, remember which operand we are guarding.
 223         MOZ_ASSERT(data.isNothing());
 224         calleeGuardOperand = reader.objOperandId();
 225         uint32_t targetOffset = reader.stubOffset();
 226         (void)reader.stubOffset();  // nargsAndFlags
 227         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
 228         target = reinterpret_cast<JSFunction*>(rawTarget);
 229         break;
 230       }
 231       case CacheOp::GuardFunctionScript: {
 232         MOZ_ASSERT(data.isNothing());
 233         calleeGuardOperand = reader.objOperandId();
 234         uint32_t targetOffset = reader.stubOffset();
 235         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
 236         target = reinterpret_cast<BaseScript*>(rawTarget)->function();
 237         (void)reader.stubOffset();  // nargsAndFlags
 238         break;
 239       }
 240       case CacheOp::CallScriptedFunction: {
 241         // If we see a call, check if `callee` is the previously guarded
 242         // operand. If it is, we know the target and can inline.
 243         ObjOperandId calleeOperand = reader.objOperandId();
 244         mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
 245         flags = reader.callFlags();
 246         mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate();
 247         MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy);
 248
 249         if (calleeOperand == calleeGuardOperand) {
 250           MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
 251           MOZ_ASSERT(data.isNothing());
 252           data.emplace();
 253           data->endOfSharedPrefix = opStart;
 254         }
 255         break;
 256       }
 257       case CacheOp::CallInlinedFunction: {
 258         ObjOperandId calleeOperand = reader.objOperandId();
 259         mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
 260         uint32_t icScriptOffset = reader.stubOffset();
 261         flags = reader.callFlags();
 262         mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate();
 263         MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy);
 264
 265         if (calleeOperand == calleeGuardOperand) {
 266           MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
 267           MOZ_ASSERT(data.isNothing());
 268           data.emplace();
 269           data->endOfSharedPrefix = opStart;
 270           uintptr_t rawICScript =
 271               stubInfo->getStubRawWord(stubData, icScriptOffset);
 272           data->icScript = reinterpret_cast<ICScript*>(rawICScript);
 273         }
 274         break;
 275       }
 276       default:
 277         if (!opInfo.transpile) {
 278           return mozilla::Nothing();
 279         }
 280         if (data.isSome()) {
 281           MOZ_ASSERT(op == CacheOp::ReturnFromIC);
 282         }
 283         reader.skip(argLength);
 284         break;
 285     }
 286     MOZ_ASSERT(argStart + argLength == reader.currentPosition());
 287   }
 288
 289   if (data.isSome()) {
 290     // Warp only supports inlining Standard and FunCall calls.
 291     if (flags.getArgFormat() != CallFlags::Standard &&
 292         flags.getArgFormat() != CallFlags::FunCall) {
 293       return mozilla::Nothing();
 294     }
 295     data->calleeOperand = calleeGuardOperand;
 296     data->callFlags = flags;
 297     data->target = target;
 298   }
 299   return data;
 300 }
 301
 302 Maybe<InlinableGetterData> FindInlinableGetterData(ICCacheIRStub* stub) {
 303   Maybe<InlinableGetterData> data;
 304
 305   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 306   const uint8_t* stubData = stub->stubDataStart();
 307
 308   CacheIRReader reader(stubInfo);
 309   while (reader.more()) {
 310     const uint8_t* opStart = reader.currentPosition();
 311
 312     CacheOp op = reader.readOp();
 313     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 314     uint32_t argLength = opInfo.argLength;
 315     mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
 316
 317     switch (op) {
 318       case CacheOp::CallScriptedGetterResult: {
 319         data.emplace();
 320         data->receiverOperand = reader.valOperandId();
 321
 322         uint32_t getterOffset = reader.stubOffset();
 323         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
 324         data->target = reinterpret_cast<JSFunction*>(rawTarget);
 325
 326         data->sameRealm = reader.readBool();
 327         (void)reader.stubOffset();  // nargsAndFlags
 328
 329         data->endOfSharedPrefix = opStart;
 330         break;
 331       }
 332       case CacheOp::CallInlinedGetterResult: {
 333         data.emplace();
 334         data->receiverOperand = reader.valOperandId();
 335
 336         uint32_t getterOffset = reader.stubOffset();
 337         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
 338         data->target = reinterpret_cast<JSFunction*>(rawTarget);
 339
 340         uint32_t icScriptOffset = reader.stubOffset();
 341         uintptr_t rawICScript =
 342             stubInfo->getStubRawWord(stubData, icScriptOffset);
 343         data->icScript = reinterpret_cast<ICScript*>(rawICScript);
 344
 345         data->sameRealm = reader.readBool();
 346         (void)reader.stubOffset();  // nargsAndFlags
 347
 348         data->endOfSharedPrefix = opStart;
 349         break;
 350       }
 351       default:
 352         if (!opInfo.transpile) {
 353           return mozilla::Nothing();
 354         }
 355         if (data.isSome()) {
 356           MOZ_ASSERT(op == CacheOp::ReturnFromIC);
 357         }
 358         reader.skip(argLength);
 359         break;
 360     }
 361     MOZ_ASSERT(argStart + argLength == reader.currentPosition());
 362   }
 363
 364   return data;
 365 }
 366
 367 Maybe<InlinableSetterData> FindInlinableSetterData(ICCacheIRStub* stub) {
 368   Maybe<InlinableSetterData> data;
 369
 370   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 371   const uint8_t* stubData = stub->stubDataStart();
 372
 373   CacheIRReader reader(stubInfo);
 374   while (reader.more()) {
 375     const uint8_t* opStart = reader.currentPosition();
 376
 377     CacheOp op = reader.readOp();
 378     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 379     uint32_t argLength = opInfo.argLength;
 380     mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
 381
 382     switch (op) {
 383       case CacheOp::CallScriptedSetter: {
 384         data.emplace();
 385         data->receiverOperand = reader.objOperandId();
 386
 387         uint32_t setterOffset = reader.stubOffset();
 388         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
 389         data->target = reinterpret_cast<JSFunction*>(rawTarget);
 390
 391         data->rhsOperand = reader.valOperandId();
 392         data->sameRealm = reader.readBool();
 393         (void)reader.stubOffset();  // nargsAndFlags
 394
 395         data->endOfSharedPrefix = opStart;
 396         break;
 397       }
 398       case CacheOp::CallInlinedSetter: {
 399         data.emplace();
 400         data->receiverOperand = reader.objOperandId();
 401
 402         uint32_t setterOffset = reader.stubOffset();
 403         uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
 404         data->target = reinterpret_cast<JSFunction*>(rawTarget);
 405
 406         data->rhsOperand = reader.valOperandId();
 407
 408         uint32_t icScriptOffset = reader.stubOffset();
 409         uintptr_t rawICScript =
 410             stubInfo->getStubRawWord(stubData, icScriptOffset);
 411         data->icScript = reinterpret_cast<ICScript*>(rawICScript);
 412
 413         data->sameRealm = reader.readBool();
 414         (void)reader.stubOffset();  // nargsAndFlags
 415
 416         data->endOfSharedPrefix = opStart;
 417         break;
 418       }
 419       default:
 420         if (!opInfo.transpile) {
 421           return mozilla::Nothing();
 422         }
 423         if (data.isSome()) {
 424           MOZ_ASSERT(op == CacheOp::ReturnFromIC);
 425         }
 426         reader.skip(argLength);
 427         break;
 428     }
 429     MOZ_ASSERT(argStart + argLength == reader.currentPosition());
 430   }
 431
 432   return data;
 433 }
 434
 435 // Return the maximum number of actual arguments that will be passed to the
 436 // target function. This may be an overapproximation, for example when inlining
 437 // js::fun_call we may omit an argument.
 438 static uint32_t GetMaxCalleeNumActuals(BytecodeLocation loc) {
 439   switch (loc.getOp()) {
 440     case JSOp::GetProp:
 441     case JSOp::GetElem:
 442       // Getters do not pass arguments.
 443       return 0;
 444
 445     case JSOp::SetProp:
 446     case JSOp::StrictSetProp:
 447       // Setters pass 1 argument.
 448       return 1;
 449
 450     case JSOp::Call:
 451     case JSOp::CallContent:
 452     case JSOp::CallIgnoresRv:
 453     case JSOp::CallIter:
 454     case JSOp::CallContentIter:
 455     case JSOp::New:
 456     case JSOp::NewContent:
 457     case JSOp::SuperCall:
 458       return loc.getCallArgc();
 459
 460     default:
 461       MOZ_CRASH("Unsupported op");
 462   }
 463 }
 464
 465 /*static*/
 466 bool TrialInliner::IsValidInliningOp(JSOp op) {
 467   switch (op) {
 468     case JSOp::GetProp:
 469     case JSOp::GetElem:
 470     case JSOp::SetProp:
 471     case JSOp::StrictSetProp:
 472     case JSOp::Call:
 473     case JSOp::CallContent:
 474     case JSOp::CallIgnoresRv:
 475     case JSOp::CallIter:
 476     case JSOp::CallContentIter:
 477     case JSOp::New:
 478     case JSOp::NewContent:
 479     case JSOp::SuperCall:
 480       return true;
 481     default:
 482       break;
 483   }
 484   return false;
 485 }
 486
 487 /*static*/
 488 bool TrialInliner::canInline(JSFunction* target, HandleScript caller,
 489                              BytecodeLocation loc) {
 490   if (!target->hasJitScript()) {
 491     JitSpew(JitSpew_WarpTrialInlining, "SKIP: no JIT script");
 492     return false;
 493   }
 494   JSScript* script = target->nonLazyScript();
 495   if (!script->jitScript()->hasBaselineScript()) {
 496     JitSpew(JitSpew_WarpTrialInlining, "SKIP: no BaselineScript");
 497     return false;
 498   }
 499   if (script->uninlineable()) {
 500     JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag");
 501     return false;
 502   }
 503   if (!script->canIonCompile()) {
 504     JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile");
 505     return false;
 506   }
 507   if (script->isDebuggee()) {
 508     JitSpew(JitSpew_WarpTrialInlining, "SKIP: is debuggee");
 509     return false;
 510   }
 511   // Don't inline cross-realm calls.
 512   if (target->realm() != caller->realm()) {
 513     JitSpew(JitSpew_WarpTrialInlining, "SKIP: cross-realm call");
 514     return false;
 515   }
 516   if (JitOptions.onlyInlineSelfHosted && !script->selfHosted()) {
 517     JitSpew(JitSpew_WarpTrialInlining, "SKIP: only inlining self hosted");
 518     return false;
 519   }
 520   if (!IsValidInliningOp(loc.getOp())) {
 521     JitSpew(JitSpew_WarpTrialInlining, "SKIP: non inlineable op");
 522     return false;
 523   }
 524
 525   uint32_t maxCalleeNumActuals = GetMaxCalleeNumActuals(loc);
 526   if (maxCalleeNumActuals > ArgumentsObject::MaxInlinedArgs) {
 527     if (script->needsArgsObj()) {
 528       JitSpew(JitSpew_WarpTrialInlining,
 529               "SKIP: needs arguments object with %u actual args (maximum %u)",
 530               maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs);
 531       return false;
 532     }
 533     // The GetArgument(n) intrinsic in self-hosted code uses MGetInlinedArgument
 534     // too, so the same limit applies.
 535     if (script->usesArgumentsIntrinsics()) {
 536       JitSpew(JitSpew_WarpTrialInlining,
 537               "SKIP: uses GetArgument(i) with %u actual args (maximum %u)",
 538               maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs);
 539       return false;
 540     }
 541   }
 542
 543   if (TooManyFormalArguments(target->nargs())) {
 544     JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u",
 545             unsigned(target->nargs()));
 546     return false;
 547   }
 548
 549   if (TooManyFormalArguments(maxCalleeNumActuals)) {
 550     JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u",
 551             unsigned(loc.getCallArgc()));
 552     return false;
 553   }
 554
 555   return true;
 556 }
 557
 558 static bool ShouldUseMonomorphicInlining(JSScript* targetScript) {
 559   switch (JitOptions.monomorphicInlining) {
 560     case UseMonomorphicInlining::Default:
 561       // Use heuristics below.
 562       break;
 563     case UseMonomorphicInlining::Always:
 564       return true;
 565     case UseMonomorphicInlining::Never:
 566       return false;
 567   }
 568
 569   JitScript* jitScript = targetScript->jitScript();
 570   ICScript* icScript = jitScript->icScript();
 571
 572   // Check for any ICs which are not monomorphic. The observation here is that
 573   // trial inlining can help us a lot in cases where it lets us further
 574   // specialize a script. But if it's already monomorphic, it's unlikely that
 575   // we will see significant specialization wins from trial inlining, so we
 576   // can use a cheaper and simpler inlining strategy.
 577   for (size_t i = 0; i < icScript->numICEntries(); i++) {
 578     ICEntry& entry = icScript->icEntry(i);
 579     ICFallbackStub* fallback = icScript->fallbackStub(i);
 580     if (fallback->enteredCount() > 0 ||
 581         fallback->state().mode() != ICState::Mode::Specialized) {
 582       return false;
 583     }
 584
 585     ICStub* firstStub = entry.firstStub();
 586     if (firstStub != fallback) {
 587       for (ICStub* next = firstStub->toCacheIRStub()->next(); next;
 588            next = next->maybeNext()) {
 589         if (next->enteredCount() != 0) {
 590           return false;
 591         }
 592       }
 593     }
 594   }
 595
 596   return true;
 597 }
 598
 599 TrialInliningDecision TrialInliner::getInliningDecision(JSFunction* target,
 600                                                         ICCacheIRStub* stub,
 601                                                         BytecodeLocation loc) {
 602 #ifdef JS_JITSPEW
 603   if (JitSpewEnabled(JitSpew_WarpTrialInlining)) {
 604     BaseScript* baseScript =
 605         target->hasBaseScript() ? target->baseScript() : nullptr;
 606
 607     UniqueChars funName;
 608     if (target->maybePartialDisplayAtom()) {
 609       funName = AtomToPrintableString(cx(), target->maybePartialDisplayAtom());
 610     }
 611
 612     JitSpew(JitSpew_WarpTrialInlining,
 613             "Inlining candidate JSOp::%s (offset=%u): callee script '%s' "
 614             "(%s:%u:%u)",
 615             CodeName(loc.getOp()), loc.bytecodeToOffset(script_),
 616             funName ? funName.get() : "<unnamed>",
 617             baseScript ? baseScript->filename() : "<not-scripted>",
 618             baseScript ? baseScript->lineno() : 0,
 619             baseScript ? baseScript->column().oneOriginValue() : 0);
 620     JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
 621   }
 622 #endif
 623
 624   if (!canInline(target, script_, loc)) {
 625     return TrialInliningDecision::NoInline;
 626   }
 627
 628   // Don't inline (direct) recursive calls. This still allows recursion if
 629   // called through another function (f => g => f).
 630   JSScript* targetScript = target->nonLazyScript();
 631   if (script_ == targetScript) {
 632     JitSpew(JitSpew_WarpTrialInlining, "SKIP: recursion");
 633     return TrialInliningDecision::NoInline;
 634   }
 635
 636   // Don't inline if the callee has a loop that was hot enough to enter Warp
 637   // via OSR. This helps prevent getting stuck in Baseline code for a long time.
 638   if (targetScript->jitScript()->hadIonOSR()) {
 639     JitSpew(JitSpew_WarpTrialInlining, "SKIP: had OSR");
 640     return TrialInliningDecision::NoInline;
 641   }
 642
 643   // Ensure the total bytecode size does not exceed ionMaxScriptSize.
 644   size_t newTotalSize =
 645       inliningRootTotalBytecodeSize() + targetScript->length();
 646   if (newTotalSize > JitOptions.ionMaxScriptSize) {
 647     JitSpew(JitSpew_WarpTrialInlining, "SKIP: total size too big");
 648     return TrialInliningDecision::NoInline;
 649   }
 650
 651   uint32_t entryCount = stub->enteredCount();
 652   if (entryCount < JitOptions.inliningEntryThreshold) {
 653     JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)",
 654             unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold));
 655     return TrialInliningDecision::NoInline;
 656   }
 657
 658   if (!JitOptions.isSmallFunction(targetScript)) {
 659     if (!targetScript->isInlinableLargeFunction()) {
 660       JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)",
 661               unsigned(targetScript->length()),
 662               unsigned(JitOptions.smallFunctionMaxBytecodeLength));
 663       return TrialInliningDecision::NoInline;
 664     }
 665
 666     JitSpew(JitSpew_WarpTrialInlining,
 667             "INFO: Ignored length (%u) of InlinableLargeFunction",
 668             unsigned(targetScript->length()));
 669   }
 670
 671   // Decide between trial inlining or monomorphic inlining.
 672   if (!ShouldUseMonomorphicInlining(targetScript)) {
 673     return TrialInliningDecision::Inline;
 674   }
 675
 676   JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
 677   JitSpew(JitSpew_WarpTrialInlining, "SUCCESS: Inlined monomorphically");
 678   return TrialInliningDecision::MonomorphicInline;
 679 }
 680
 681 ICScript* TrialInliner::createInlinedICScript(JSFunction* target,
 682                                               BytecodeLocation loc) {
 683   MOZ_ASSERT(target->hasJitEntry());
 684   MOZ_ASSERT(target->hasJitScript());
 685
 686   InliningRoot* root = getOrCreateInliningRoot();
 687   if (!root) {
 688     return nullptr;
 689   }
 690
 691   JSScript* targetScript = target->baseScript()->asJSScript();
 692
 693   // We don't have to check for overflow here because we have already
 694   // successfully allocated an ICScript with this number of entries
 695   // when creating the JitScript for the target function, and we
 696   // checked for overflow then.
 697   uint32_t fallbackStubsOffset =
 698       sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry);
 699   uint32_t allocSize = fallbackStubsOffset +
 700                        targetScript->numICEntries() * sizeof(ICFallbackStub);
 701
 702   void* raw = cx()->pod_malloc<uint8_t>(allocSize);
 703   MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0);
 704   if (!raw) {
 705     return nullptr;
 706   }
 707
 708   uint32_t initialWarmUpCount = JitOptions.trialInliningInitialWarmUpCount;
 709
 710   uint32_t depth = icScript_->depth() + 1;
 711   UniquePtr<ICScript> inlinedICScript(
 712       new (raw) ICScript(initialWarmUpCount, fallbackStubsOffset, allocSize,
 713                          depth, targetScript->length(), root));
 714
 715   inlinedICScript->initICEntries(cx(), targetScript);
 716
 717   uint32_t pcOffset = loc.bytecodeToOffset(script_);
 718   ICScript* result = inlinedICScript.get();
 719   if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) {
 720     return nullptr;
 721   }
 722   MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries());
 723
 724   root->addToTotalBytecodeSize(targetScript->length());
 725
 726   JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
 727   JitSpew(JitSpew_WarpTrialInlining,
 728           "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_, result);
 729
 730   return result;
 731 }
 732
 733 bool TrialInliner::maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback,
 734                                    BytecodeLocation loc) {
 735   ICCacheIRStub* stub = maybeSingleStub(entry);
 736   if (!stub) {
 737 #ifdef JS_JITSPEW
 738     if (fallback->numOptimizedStubs() > 1) {
 739       JitSpew(JitSpew_WarpTrialInlining,
 740               "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc.getOp()),
 741               fallback->pcOffset());
 742       JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
 743       JitSpew(JitSpew_WarpTrialInlining, "SKIP: Polymorphic (%u stubs)",
 744               (unsigned)fallback->numOptimizedStubs());
 745     }
 746 #endif
 747     return true;
 748   }
 749
 750   MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
 751
 752   // Look for a CallScriptedFunction with a known target.
 753   Maybe<InlinableCallData> data = FindInlinableCallData(stub);
 754   if (data.isNothing()) {
 755     return true;
 756   }
 757
 758   MOZ_ASSERT(!data->icScript);
 759
 760   TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
 761   // Decide whether to inline the target.
 762   if (inlining == TrialInliningDecision::NoInline) {
 763     return true;
 764   }
 765
 766   if (inlining == TrialInliningDecision::MonomorphicInline) {
 767     fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
 768     return true;
 769   }
 770
 771   ICScript* newICScript = createInlinedICScript(data->target, loc);
 772   if (!newICScript) {
 773     return false;
 774   }
 775
 776   CacheIRWriter writer(cx());
 777   Int32OperandId argcId(writer.setInputOperandId(0));
 778   cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
 779
 780   writer.callInlinedFunction(data->calleeOperand, argcId, newICScript,
 781                              data->callFlags,
 782                              ClampFixedArgc(loc.getCallArgc()));
 783   writer.returnFromIC();
 784
 785   return replaceICStub(entry, fallback, writer, CacheKind::Call);
 786 }
 787
 788 bool TrialInliner::maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback,
 789                                      BytecodeLocation loc, CacheKind kind) {
 790   ICCacheIRStub* stub = maybeSingleStub(entry);
 791   if (!stub) {
 792     return true;
 793   }
 794
 795   MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
 796
 797   Maybe<InlinableGetterData> data = FindInlinableGetterData(stub);
 798   if (data.isNothing()) {
 799     return true;
 800   }
 801
 802   MOZ_ASSERT(!data->icScript);
 803
 804   TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
 805   // Decide whether to inline the target.
 806   if (inlining == TrialInliningDecision::NoInline) {
 807     return true;
 808   }
 809
 810   if (inlining == TrialInliningDecision::MonomorphicInline) {
 811     fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
 812     return true;
 813   }
 814
 815   ICScript* newICScript = createInlinedICScript(data->target, loc);
 816   if (!newICScript) {
 817     return false;
 818   }
 819
 820   CacheIRWriter writer(cx());
 821   ValOperandId valId(writer.setInputOperandId(0));
 822   if (kind == CacheKind::GetElem) {
 823     // Register the key operand.
 824     writer.setInputOperandId(1);
 825   }
 826   cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
 827
 828   writer.callInlinedGetterResult(data->receiverOperand, data->target,
 829                                  newICScript, data->sameRealm);
 830   writer.returnFromIC();
 831
 832   return replaceICStub(entry, fallback, writer, kind);
 833 }
 834
 835 bool TrialInliner::maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback,
 836                                      BytecodeLocation loc, CacheKind kind) {
 837   ICCacheIRStub* stub = maybeSingleStub(entry);
 838   if (!stub) {
 839     return true;
 840   }
 841
 842   MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
 843
 844   Maybe<InlinableSetterData> data = FindInlinableSetterData(stub);
 845   if (data.isNothing()) {
 846     return true;
 847   }
 848
 849   MOZ_ASSERT(!data->icScript);
 850
 851   TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
 852   // Decide whether to inline the target.
 853   if (inlining == TrialInliningDecision::NoInline) {
 854     return true;
 855   }
 856
 857   if (inlining == TrialInliningDecision::MonomorphicInline) {
 858     fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
 859     return true;
 860   }
 861
 862   ICScript* newICScript = createInlinedICScript(data->target, loc);
 863   if (!newICScript) {
 864     return false;
 865   }
 866
 867   CacheIRWriter writer(cx());
 868   ValOperandId objValId(writer.setInputOperandId(0));
 869   ValOperandId rhsValId(writer.setInputOperandId(1));
 870   cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
 871
 872   writer.callInlinedSetter(data->receiverOperand, data->target,
 873                            data->rhsOperand, newICScript, data->sameRealm);
 874   writer.returnFromIC();
 875
 876   return replaceICStub(entry, fallback, writer, kind);
 877 }
 878
 879 bool TrialInliner::tryInlining() {
 880   uint32_t numICEntries = icScript_->numICEntries();
 881   BytecodeLocation startLoc = script_->location();
 882
 883   for (uint32_t icIndex = 0; icIndex < numICEntries; icIndex++) {
 884     ICEntry& entry = icScript_->icEntry(icIndex);
 885     ICFallbackStub* fallback = icScript_->fallbackStub(icIndex);
 886
 887     if (!TryFoldingStubs(cx(), fallback, script_, icScript_)) {
 888       return false;
 889     }
 890
 891     BytecodeLocation loc =
 892         startLoc + BytecodeLocationOffset(fallback->pcOffset());
 893     JSOp op = loc.getOp();
 894     switch (op) {
 895       case JSOp::Call:
 896       case JSOp::CallContent:
 897       case JSOp::CallIgnoresRv:
 898       case JSOp::CallIter:
 899       case JSOp::CallContentIter:
 900       case JSOp::New:
 901       case JSOp::NewContent:
 902       case JSOp::SuperCall:
 903         if (!maybeInlineCall(entry, fallback, loc)) {
 904           return false;
 905         }
 906         break;
 907       case JSOp::GetProp:
 908         if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetProp)) {
 909           return false;
 910         }
 911         break;
 912       case JSOp::GetElem:
 913         if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetElem)) {
 914           return false;
 915         }
 916         break;
 917       case JSOp::SetProp:
 918       case JSOp::StrictSetProp:
 919         if (!maybeInlineSetter(entry, fallback, loc, CacheKind::SetProp)) {
 920           return false;
 921         }
 922         break;
 923       default:
 924         break;
 925     }
 926   }
 927
 928   return true;
 929 }
 930
 931 InliningRoot* TrialInliner::maybeGetInliningRoot() const {
 932   if (auto* root = icScript_->inliningRoot()) {
 933     return root;
 934   }
 935
 936   MOZ_ASSERT(!icScript_->isInlined());
 937   return script_->jitScript()->inliningRoot();
 938 }
 939
 940 InliningRoot* TrialInliner::getOrCreateInliningRoot() {
 941   if (auto* root = maybeGetInliningRoot()) {
 942     return root;
 943   }
 944   return script_->jitScript()->getOrCreateInliningRoot(cx(), script_);
 945 }
 946
 947 size_t TrialInliner::inliningRootTotalBytecodeSize() const {
 948   if (auto* root = maybeGetInliningRoot()) {
 949     return root->totalBytecodeSize();
 950   }
 951   return script_->length();
 952 }
 953
 954 bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) {
 955   return inlinedScripts_.append(std::move(icScript));
 956 }
 957
 958 void InliningRoot::trace(JSTracer* trc) {
 959   TraceEdge(trc, &owningScript_, "inlining-root-owning-script");
 960   for (auto& inlinedScript : inlinedScripts_) {
 961     inlinedScript->trace(trc);
 962   }
 963 }
 964
 965 bool InliningRoot::traceWeak(JSTracer* trc) {
 966   bool allSurvived = true;
 967   for (auto& inlinedScript : inlinedScripts_) {
 968     if (!inlinedScript->traceWeak(trc)) {
 969       allSurvived = false;
 970     }
 971   }
 972   return allSurvived;
 973 }
 974
 975 void InliningRoot::purgeInactiveICScripts() {
 976   mozilla::DebugOnly<uint32_t> totalSize = owningScript_->length();
 977
 978   for (auto& inlinedScript : inlinedScripts_) {
 979     if (inlinedScript->active()) {
 980       totalSize += inlinedScript->bytecodeSize();
 981     } else {
 982       MOZ_ASSERT(inlinedScript->bytecodeSize() < totalBytecodeSize_);
 983       totalBytecodeSize_ -= inlinedScript->bytecodeSize();
 984     }
 985   }
 986
 987   MOZ_ASSERT(totalBytecodeSize_ == totalSize);
 988
 989   Zone* zone = owningScript_->zone();
 990
 991   inlinedScripts_.eraseIf([zone](auto& inlinedScript) {
 992     if (inlinedScript->active()) {
 993       return false;
 994     }
 995     inlinedScript->prepareForDestruction(zone);
 996     return true;
 997   });
 998 }
 999
1000 }  // namespace jit
1001 }  // namespace js