1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "jit/TrialInlining.h"
9 #include "mozilla/DebugOnly.h"
11 #include "jit/BaselineCacheIRCompiler.h"
12 #include "jit/BaselineFrame.h"
13 #include "jit/BaselineIC.h"
14 #include "jit/CacheIRCloner.h"
15 #include "jit/CacheIRHealth.h"
16 #include "jit/CacheIRWriter.h"
17 #include "jit/Ion.h" // TooManyFormalArguments
19 #include "vm/BytecodeLocation-inl.h"
26 bool DoTrialInlining(JSContext
* cx
, BaselineFrame
* frame
) {
27 RootedScript
script(cx
, frame
->script());
28 ICScript
* icScript
= frame
->icScript();
29 bool isRecursive
= icScript
->depth() > 0;
31 #ifdef JS_CACHEIR_SPEW
32 if (cx
->spewer().enabled(cx
, script
, SpewChannel::CacheIRHealthReport
)) {
33 for (uint32_t i
= 0; i
< icScript
->numICEntries(); i
++) {
34 ICEntry
& entry
= icScript
->icEntry(i
);
35 ICFallbackStub
* fallbackStub
= icScript
->fallbackStub(i
);
37 // If the IC is megamorphic or generic, then we have already
38 // spewed the IC report on transition.
39 if (!(uint8_t(fallbackStub
->state().mode()) > 0)) {
40 jit::ICStub
* stub
= entry
.firstStub();
41 bool sawNonZeroCount
= false;
42 while (!stub
->isFallback()) {
43 uint32_t count
= stub
->enteredCount();
44 if (count
> 0 && sawNonZeroCount
) {
46 cih
.healthReportForIC(cx
, &entry
, fallbackStub
, script
,
47 SpewContext::TrialInlining
);
51 if (count
> 0 && !sawNonZeroCount
) {
52 sawNonZeroCount
= true;
55 stub
= stub
->toCacheIRStub()->next();
62 if (!script
->canIonCompile()) {
66 // Baseline shouldn't attempt trial inlining in scripts that are too large.
67 MOZ_ASSERT_IF(JitOptions
.limitScriptSize
,
68 script
->length() <= JitOptions
.ionMaxScriptSize
);
70 const uint32_t MAX_INLINING_DEPTH
= 4;
71 if (icScript
->depth() > MAX_INLINING_DEPTH
) {
75 InliningRoot
* root
= isRecursive
? icScript
->inliningRoot()
76 : script
->jitScript()->inliningRoot();
77 if (JitSpewEnabled(JitSpew_WarpTrialInlining
)) {
78 // Eagerly create the inlining root when it's used in the spew output.
80 MOZ_ASSERT(!isRecursive
);
81 root
= script
->jitScript()->getOrCreateInliningRoot(cx
, script
);
87 if (script
->function() && script
->function()->fullDisplayAtom()) {
89 AtomToPrintableString(cx
, script
->function()->fullDisplayAtom());
93 JitSpew_WarpTrialInlining
,
94 "Trial inlining for %s script '%s' (%s:%u:%u (%p)) (inliningRoot=%p)",
95 (isRecursive
? "inner" : "outer"),
96 funName
? funName
.get() : "<unnamed>", script
->filename(),
97 script
->lineno(), script
->column().oneOriginValue(), frame
->script(),
99 JitSpewIndent
spewIndent(JitSpew_WarpTrialInlining
);
102 TrialInliner
inliner(cx
, script
, icScript
);
103 return inliner
.tryInlining();
106 void TrialInliner::cloneSharedPrefix(ICCacheIRStub
* stub
,
107 const uint8_t* endOfPrefix
,
108 CacheIRWriter
& writer
) {
109 CacheIRReader
reader(stub
->stubInfo());
110 CacheIRCloner
cloner(stub
);
111 while (reader
.currentPosition() < endOfPrefix
) {
112 CacheOp op
= reader
.readOp();
113 cloner
.cloneOp(op
, reader
, writer
);
117 bool TrialInliner::replaceICStub(ICEntry
& entry
, ICFallbackStub
* fallback
,
118 CacheIRWriter
& writer
, CacheKind kind
) {
119 MOZ_ASSERT(fallback
->trialInliningState() == TrialInliningState::Candidate
);
121 fallback
->discardStubs(cx()->zone(), &entry
);
123 // Note: AttachBaselineCacheIRStub never throws an exception.
124 ICAttachResult result
= AttachBaselineCacheIRStub(
125 cx(), writer
, kind
, script_
, icScript_
, fallback
, "TrialInline");
126 if (result
== ICAttachResult::Attached
) {
127 MOZ_ASSERT(fallback
->trialInliningState() == TrialInliningState::Inlined
);
131 MOZ_ASSERT(fallback
->trialInliningState() == TrialInliningState::Candidate
);
132 icScript_
->removeInlinedChild(fallback
->pcOffset());
134 if (result
== ICAttachResult::OOM
) {
135 ReportOutOfMemory(cx());
139 // We failed to attach a new IC stub due to CacheIR size limits. Disable trial
140 // inlining for this location and return true.
141 MOZ_ASSERT(result
== ICAttachResult::TooLarge
);
142 fallback
->setTrialInliningState(TrialInliningState::Failure
);
146 ICCacheIRStub
* TrialInliner::maybeSingleStub(const ICEntry
& entry
) {
147 // Look for a single non-fallback stub followed by stubs with entered-count 0.
148 // Allow one optimized stub before the fallback stub to support the
149 // CallIRGenerator::emitCalleeGuard optimization where we first try a
150 // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
151 ICStub
* stub
= entry
.firstStub();
152 if (stub
->isFallback()) {
155 ICStub
* next
= stub
->toCacheIRStub()->next();
156 if (next
->enteredCount() != 0) {
160 ICFallbackStub
* fallback
= nullptr;
161 if (next
->isFallback()) {
162 fallback
= next
->toFallbackStub();
164 ICStub
* nextNext
= next
->toCacheIRStub()->next();
165 if (!nextNext
->isFallback() || nextNext
->enteredCount() != 0) {
168 fallback
= nextNext
->toFallbackStub();
171 if (fallback
->trialInliningState() != TrialInliningState::Candidate
) {
175 return stub
->toCacheIRStub();
178 Maybe
<InlinableOpData
> FindInlinableOpData(ICCacheIRStub
* stub
,
179 BytecodeLocation loc
) {
180 if (loc
.isInvokeOp()) {
181 Maybe
<InlinableCallData
> call
= FindInlinableCallData(stub
);
186 if (loc
.isGetPropOp() || loc
.isGetElemOp()) {
187 Maybe
<InlinableGetterData
> getter
= FindInlinableGetterData(stub
);
188 if (getter
.isSome()) {
192 if (loc
.isSetPropOp()) {
193 Maybe
<InlinableSetterData
> setter
= FindInlinableSetterData(stub
);
194 if (setter
.isSome()) {
198 return mozilla::Nothing();
201 Maybe
<InlinableCallData
> FindInlinableCallData(ICCacheIRStub
* stub
) {
202 Maybe
<InlinableCallData
> data
;
204 const CacheIRStubInfo
* stubInfo
= stub
->stubInfo();
205 const uint8_t* stubData
= stub
->stubDataStart();
207 ObjOperandId calleeGuardOperand
;
209 JSFunction
* target
= nullptr;
211 CacheIRReader
reader(stubInfo
);
212 while (reader
.more()) {
213 const uint8_t* opStart
= reader
.currentPosition();
215 CacheOp op
= reader
.readOp();
216 CacheIROpInfo opInfo
= CacheIROpInfos
[size_t(op
)];
217 uint32_t argLength
= opInfo
.argLength
;
218 mozilla::DebugOnly
<const uint8_t*> argStart
= reader
.currentPosition();
221 case CacheOp::GuardSpecificFunction
: {
222 // If we see a guard, remember which operand we are guarding.
223 MOZ_ASSERT(data
.isNothing());
224 calleeGuardOperand
= reader
.objOperandId();
225 uint32_t targetOffset
= reader
.stubOffset();
226 (void)reader
.stubOffset(); // nargsAndFlags
227 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, targetOffset
);
228 target
= reinterpret_cast<JSFunction
*>(rawTarget
);
231 case CacheOp::GuardFunctionScript
: {
232 MOZ_ASSERT(data
.isNothing());
233 calleeGuardOperand
= reader
.objOperandId();
234 uint32_t targetOffset
= reader
.stubOffset();
235 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, targetOffset
);
236 target
= reinterpret_cast<BaseScript
*>(rawTarget
)->function();
237 (void)reader
.stubOffset(); // nargsAndFlags
240 case CacheOp::CallScriptedFunction
: {
241 // If we see a call, check if `callee` is the previously guarded
242 // operand. If it is, we know the target and can inline.
243 ObjOperandId calleeOperand
= reader
.objOperandId();
244 mozilla::DebugOnly
<Int32OperandId
> argcId
= reader
.int32OperandId();
245 flags
= reader
.callFlags();
246 mozilla::DebugOnly
<uint32_t> argcFixed
= reader
.uint32Immediate();
247 MOZ_ASSERT(argcFixed
<= MaxUnrolledArgCopy
);
249 if (calleeOperand
== calleeGuardOperand
) {
250 MOZ_ASSERT(static_cast<OperandId
&>(argcId
).id() == 0);
251 MOZ_ASSERT(data
.isNothing());
253 data
->endOfSharedPrefix
= opStart
;
257 case CacheOp::CallInlinedFunction
: {
258 ObjOperandId calleeOperand
= reader
.objOperandId();
259 mozilla::DebugOnly
<Int32OperandId
> argcId
= reader
.int32OperandId();
260 uint32_t icScriptOffset
= reader
.stubOffset();
261 flags
= reader
.callFlags();
262 mozilla::DebugOnly
<uint32_t> argcFixed
= reader
.uint32Immediate();
263 MOZ_ASSERT(argcFixed
<= MaxUnrolledArgCopy
);
265 if (calleeOperand
== calleeGuardOperand
) {
266 MOZ_ASSERT(static_cast<OperandId
&>(argcId
).id() == 0);
267 MOZ_ASSERT(data
.isNothing());
269 data
->endOfSharedPrefix
= opStart
;
270 uintptr_t rawICScript
=
271 stubInfo
->getStubRawWord(stubData
, icScriptOffset
);
272 data
->icScript
= reinterpret_cast<ICScript
*>(rawICScript
);
277 if (!opInfo
.transpile
) {
278 return mozilla::Nothing();
281 MOZ_ASSERT(op
== CacheOp::ReturnFromIC
);
283 reader
.skip(argLength
);
286 MOZ_ASSERT(argStart
+ argLength
== reader
.currentPosition());
290 // Warp only supports inlining Standard and FunCall calls.
291 if (flags
.getArgFormat() != CallFlags::Standard
&&
292 flags
.getArgFormat() != CallFlags::FunCall
) {
293 return mozilla::Nothing();
295 data
->calleeOperand
= calleeGuardOperand
;
296 data
->callFlags
= flags
;
297 data
->target
= target
;
302 Maybe
<InlinableGetterData
> FindInlinableGetterData(ICCacheIRStub
* stub
) {
303 Maybe
<InlinableGetterData
> data
;
305 const CacheIRStubInfo
* stubInfo
= stub
->stubInfo();
306 const uint8_t* stubData
= stub
->stubDataStart();
308 CacheIRReader
reader(stubInfo
);
309 while (reader
.more()) {
310 const uint8_t* opStart
= reader
.currentPosition();
312 CacheOp op
= reader
.readOp();
313 CacheIROpInfo opInfo
= CacheIROpInfos
[size_t(op
)];
314 uint32_t argLength
= opInfo
.argLength
;
315 mozilla::DebugOnly
<const uint8_t*> argStart
= reader
.currentPosition();
318 case CacheOp::CallScriptedGetterResult
: {
320 data
->receiverOperand
= reader
.valOperandId();
322 uint32_t getterOffset
= reader
.stubOffset();
323 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, getterOffset
);
324 data
->target
= reinterpret_cast<JSFunction
*>(rawTarget
);
326 data
->sameRealm
= reader
.readBool();
327 (void)reader
.stubOffset(); // nargsAndFlags
329 data
->endOfSharedPrefix
= opStart
;
332 case CacheOp::CallInlinedGetterResult
: {
334 data
->receiverOperand
= reader
.valOperandId();
336 uint32_t getterOffset
= reader
.stubOffset();
337 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, getterOffset
);
338 data
->target
= reinterpret_cast<JSFunction
*>(rawTarget
);
340 uint32_t icScriptOffset
= reader
.stubOffset();
341 uintptr_t rawICScript
=
342 stubInfo
->getStubRawWord(stubData
, icScriptOffset
);
343 data
->icScript
= reinterpret_cast<ICScript
*>(rawICScript
);
345 data
->sameRealm
= reader
.readBool();
346 (void)reader
.stubOffset(); // nargsAndFlags
348 data
->endOfSharedPrefix
= opStart
;
352 if (!opInfo
.transpile
) {
353 return mozilla::Nothing();
356 MOZ_ASSERT(op
== CacheOp::ReturnFromIC
);
358 reader
.skip(argLength
);
361 MOZ_ASSERT(argStart
+ argLength
== reader
.currentPosition());
367 Maybe
<InlinableSetterData
> FindInlinableSetterData(ICCacheIRStub
* stub
) {
368 Maybe
<InlinableSetterData
> data
;
370 const CacheIRStubInfo
* stubInfo
= stub
->stubInfo();
371 const uint8_t* stubData
= stub
->stubDataStart();
373 CacheIRReader
reader(stubInfo
);
374 while (reader
.more()) {
375 const uint8_t* opStart
= reader
.currentPosition();
377 CacheOp op
= reader
.readOp();
378 CacheIROpInfo opInfo
= CacheIROpInfos
[size_t(op
)];
379 uint32_t argLength
= opInfo
.argLength
;
380 mozilla::DebugOnly
<const uint8_t*> argStart
= reader
.currentPosition();
383 case CacheOp::CallScriptedSetter
: {
385 data
->receiverOperand
= reader
.objOperandId();
387 uint32_t setterOffset
= reader
.stubOffset();
388 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, setterOffset
);
389 data
->target
= reinterpret_cast<JSFunction
*>(rawTarget
);
391 data
->rhsOperand
= reader
.valOperandId();
392 data
->sameRealm
= reader
.readBool();
393 (void)reader
.stubOffset(); // nargsAndFlags
395 data
->endOfSharedPrefix
= opStart
;
398 case CacheOp::CallInlinedSetter
: {
400 data
->receiverOperand
= reader
.objOperandId();
402 uint32_t setterOffset
= reader
.stubOffset();
403 uintptr_t rawTarget
= stubInfo
->getStubRawWord(stubData
, setterOffset
);
404 data
->target
= reinterpret_cast<JSFunction
*>(rawTarget
);
406 data
->rhsOperand
= reader
.valOperandId();
408 uint32_t icScriptOffset
= reader
.stubOffset();
409 uintptr_t rawICScript
=
410 stubInfo
->getStubRawWord(stubData
, icScriptOffset
);
411 data
->icScript
= reinterpret_cast<ICScript
*>(rawICScript
);
413 data
->sameRealm
= reader
.readBool();
414 (void)reader
.stubOffset(); // nargsAndFlags
416 data
->endOfSharedPrefix
= opStart
;
420 if (!opInfo
.transpile
) {
421 return mozilla::Nothing();
424 MOZ_ASSERT(op
== CacheOp::ReturnFromIC
);
426 reader
.skip(argLength
);
429 MOZ_ASSERT(argStart
+ argLength
== reader
.currentPosition());
435 // Return the maximum number of actual arguments that will be passed to the
436 // target function. This may be an overapproximation, for example when inlining
437 // js::fun_call we may omit an argument.
438 static uint32_t GetMaxCalleeNumActuals(BytecodeLocation loc
) {
439 switch (loc
.getOp()) {
442 // Getters do not pass arguments.
446 case JSOp::StrictSetProp
:
447 // Setters pass 1 argument.
451 case JSOp::CallContent
:
452 case JSOp::CallIgnoresRv
:
454 case JSOp::CallContentIter
:
456 case JSOp::NewContent
:
457 case JSOp::SuperCall
:
458 return loc
.getCallArgc();
461 MOZ_CRASH("Unsupported op");
466 bool TrialInliner::IsValidInliningOp(JSOp op
) {
471 case JSOp::StrictSetProp
:
473 case JSOp::CallContent
:
474 case JSOp::CallIgnoresRv
:
476 case JSOp::CallContentIter
:
478 case JSOp::NewContent
:
479 case JSOp::SuperCall
:
488 bool TrialInliner::canInline(JSFunction
* target
, HandleScript caller
,
489 BytecodeLocation loc
) {
490 if (!target
->hasJitScript()) {
491 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: no JIT script");
494 JSScript
* script
= target
->nonLazyScript();
495 if (!script
->jitScript()->hasBaselineScript()) {
496 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: no BaselineScript");
499 if (script
->uninlineable()) {
500 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: uninlineable flag");
503 if (!script
->canIonCompile()) {
504 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: can't ion-compile");
507 if (script
->isDebuggee()) {
508 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: is debuggee");
511 // Don't inline cross-realm calls.
512 if (target
->realm() != caller
->realm()) {
513 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: cross-realm call");
516 if (JitOptions
.onlyInlineSelfHosted
&& !script
->selfHosted()) {
517 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: only inlining self hosted");
520 if (!IsValidInliningOp(loc
.getOp())) {
521 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: non inlineable op");
525 uint32_t maxCalleeNumActuals
= GetMaxCalleeNumActuals(loc
);
526 if (maxCalleeNumActuals
> ArgumentsObject::MaxInlinedArgs
) {
527 if (script
->needsArgsObj()) {
528 JitSpew(JitSpew_WarpTrialInlining
,
529 "SKIP: needs arguments object with %u actual args (maximum %u)",
530 maxCalleeNumActuals
, ArgumentsObject::MaxInlinedArgs
);
533 // The GetArgument(n) intrinsic in self-hosted code uses MGetInlinedArgument
534 // too, so the same limit applies.
535 if (script
->usesArgumentsIntrinsics()) {
536 JitSpew(JitSpew_WarpTrialInlining
,
537 "SKIP: uses GetArgument(i) with %u actual args (maximum %u)",
538 maxCalleeNumActuals
, ArgumentsObject::MaxInlinedArgs
);
543 if (TooManyFormalArguments(target
->nargs())) {
544 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: Too many formal arguments: %u",
545 unsigned(target
->nargs()));
549 if (TooManyFormalArguments(maxCalleeNumActuals
)) {
550 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: argc too large: %u",
551 unsigned(loc
.getCallArgc()));
558 static bool ShouldUseMonomorphicInlining(JSScript
* targetScript
) {
559 switch (JitOptions
.monomorphicInlining
) {
560 case UseMonomorphicInlining::Default
:
561 // Use heuristics below.
563 case UseMonomorphicInlining::Always
:
565 case UseMonomorphicInlining::Never
:
569 JitScript
* jitScript
= targetScript
->jitScript();
570 ICScript
* icScript
= jitScript
->icScript();
572 // Check for any ICs which are not monomorphic. The observation here is that
573 // trial inlining can help us a lot in cases where it lets us further
574 // specialize a script. But if it's already monomorphic, it's unlikely that
575 // we will see significant specialization wins from trial inlining, so we
576 // can use a cheaper and simpler inlining strategy.
577 for (size_t i
= 0; i
< icScript
->numICEntries(); i
++) {
578 ICEntry
& entry
= icScript
->icEntry(i
);
579 ICFallbackStub
* fallback
= icScript
->fallbackStub(i
);
580 if (fallback
->enteredCount() > 0 ||
581 fallback
->state().mode() != ICState::Mode::Specialized
) {
585 ICStub
* firstStub
= entry
.firstStub();
586 if (firstStub
!= fallback
) {
587 for (ICStub
* next
= firstStub
->toCacheIRStub()->next(); next
;
588 next
= next
->maybeNext()) {
589 if (next
->enteredCount() != 0) {
599 TrialInliningDecision
TrialInliner::getInliningDecision(JSFunction
* target
,
601 BytecodeLocation loc
) {
603 if (JitSpewEnabled(JitSpew_WarpTrialInlining
)) {
604 BaseScript
* baseScript
=
605 target
->hasBaseScript() ? target
->baseScript() : nullptr;
608 if (target
->maybePartialDisplayAtom()) {
609 funName
= AtomToPrintableString(cx(), target
->maybePartialDisplayAtom());
612 JitSpew(JitSpew_WarpTrialInlining
,
613 "Inlining candidate JSOp::%s (offset=%u): callee script '%s' "
615 CodeName(loc
.getOp()), loc
.bytecodeToOffset(script_
),
616 funName
? funName
.get() : "<unnamed>",
617 baseScript
? baseScript
->filename() : "<not-scripted>",
618 baseScript
? baseScript
->lineno() : 0,
619 baseScript
? baseScript
->column().oneOriginValue() : 0);
620 JitSpewIndent
spewIndent(JitSpew_WarpTrialInlining
);
624 if (!canInline(target
, script_
, loc
)) {
625 return TrialInliningDecision::NoInline
;
628 // Don't inline (direct) recursive calls. This still allows recursion if
629 // called through another function (f => g => f).
630 JSScript
* targetScript
= target
->nonLazyScript();
631 if (script_
== targetScript
) {
632 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: recursion");
633 return TrialInliningDecision::NoInline
;
636 // Don't inline if the callee has a loop that was hot enough to enter Warp
637 // via OSR. This helps prevent getting stuck in Baseline code for a long time.
638 if (targetScript
->jitScript()->hadIonOSR()) {
639 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: had OSR");
640 return TrialInliningDecision::NoInline
;
643 // Ensure the total bytecode size does not exceed ionMaxScriptSize.
644 size_t newTotalSize
=
645 inliningRootTotalBytecodeSize() + targetScript
->length();
646 if (newTotalSize
> JitOptions
.ionMaxScriptSize
) {
647 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: total size too big");
648 return TrialInliningDecision::NoInline
;
651 uint32_t entryCount
= stub
->enteredCount();
652 if (entryCount
< JitOptions
.inliningEntryThreshold
) {
653 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: Entry count is %u (minimum %u)",
654 unsigned(entryCount
), unsigned(JitOptions
.inliningEntryThreshold
));
655 return TrialInliningDecision::NoInline
;
658 if (!JitOptions
.isSmallFunction(targetScript
)) {
659 if (!targetScript
->isInlinableLargeFunction()) {
660 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: Length is %u (maximum %u)",
661 unsigned(targetScript
->length()),
662 unsigned(JitOptions
.smallFunctionMaxBytecodeLength
));
663 return TrialInliningDecision::NoInline
;
666 JitSpew(JitSpew_WarpTrialInlining
,
667 "INFO: Ignored length (%u) of InlinableLargeFunction",
668 unsigned(targetScript
->length()));
671 // Decide between trial inlining or monomorphic inlining.
672 if (!ShouldUseMonomorphicInlining(targetScript
)) {
673 return TrialInliningDecision::Inline
;
676 JitSpewIndent
spewIndent(JitSpew_WarpTrialInlining
);
677 JitSpew(JitSpew_WarpTrialInlining
, "SUCCESS: Inlined monomorphically");
678 return TrialInliningDecision::MonomorphicInline
;
681 ICScript
* TrialInliner::createInlinedICScript(JSFunction
* target
,
682 BytecodeLocation loc
) {
683 MOZ_ASSERT(target
->hasJitEntry());
684 MOZ_ASSERT(target
->hasJitScript());
686 InliningRoot
* root
= getOrCreateInliningRoot();
691 JSScript
* targetScript
= target
->baseScript()->asJSScript();
693 // We don't have to check for overflow here because we have already
694 // successfully allocated an ICScript with this number of entries
695 // when creating the JitScript for the target function, and we
696 // checked for overflow then.
697 uint32_t fallbackStubsOffset
=
698 sizeof(ICScript
) + targetScript
->numICEntries() * sizeof(ICEntry
);
699 uint32_t allocSize
= fallbackStubsOffset
+
700 targetScript
->numICEntries() * sizeof(ICFallbackStub
);
702 void* raw
= cx()->pod_malloc
<uint8_t>(allocSize
);
703 MOZ_ASSERT(uintptr_t(raw
) % alignof(ICScript
) == 0);
708 uint32_t initialWarmUpCount
= JitOptions
.trialInliningInitialWarmUpCount
;
710 uint32_t depth
= icScript_
->depth() + 1;
711 UniquePtr
<ICScript
> inlinedICScript(
712 new (raw
) ICScript(initialWarmUpCount
, fallbackStubsOffset
, allocSize
,
713 depth
, targetScript
->length(), root
));
715 inlinedICScript
->initICEntries(cx(), targetScript
);
717 uint32_t pcOffset
= loc
.bytecodeToOffset(script_
);
718 ICScript
* result
= inlinedICScript
.get();
719 if (!icScript_
->addInlinedChild(cx(), std::move(inlinedICScript
), pcOffset
)) {
722 MOZ_ASSERT(result
->numICEntries() == targetScript
->numICEntries());
724 root
->addToTotalBytecodeSize(targetScript
->length());
726 JitSpewIndent
spewIndent(JitSpew_WarpTrialInlining
);
727 JitSpew(JitSpew_WarpTrialInlining
,
728 "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_
, result
);
733 bool TrialInliner::maybeInlineCall(ICEntry
& entry
, ICFallbackStub
* fallback
,
734 BytecodeLocation loc
) {
735 ICCacheIRStub
* stub
= maybeSingleStub(entry
);
738 if (fallback
->numOptimizedStubs() > 1) {
739 JitSpew(JitSpew_WarpTrialInlining
,
740 "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc
.getOp()),
741 fallback
->pcOffset());
742 JitSpewIndent
spewIndent(JitSpew_WarpTrialInlining
);
743 JitSpew(JitSpew_WarpTrialInlining
, "SKIP: Polymorphic (%u stubs)",
744 (unsigned)fallback
->numOptimizedStubs());
750 MOZ_ASSERT(!icScript_
->hasInlinedChild(fallback
->pcOffset()));
752 // Look for a CallScriptedFunction with a known target.
753 Maybe
<InlinableCallData
> data
= FindInlinableCallData(stub
);
754 if (data
.isNothing()) {
758 MOZ_ASSERT(!data
->icScript
);
760 TrialInliningDecision inlining
= getInliningDecision(data
->target
, stub
, loc
);
761 // Decide whether to inline the target.
762 if (inlining
== TrialInliningDecision::NoInline
) {
766 if (inlining
== TrialInliningDecision::MonomorphicInline
) {
767 fallback
->setTrialInliningState(TrialInliningState::MonomorphicInlined
);
771 ICScript
* newICScript
= createInlinedICScript(data
->target
, loc
);
776 CacheIRWriter
writer(cx());
777 Int32OperandId
argcId(writer
.setInputOperandId(0));
778 cloneSharedPrefix(stub
, data
->endOfSharedPrefix
, writer
);
780 writer
.callInlinedFunction(data
->calleeOperand
, argcId
, newICScript
,
782 ClampFixedArgc(loc
.getCallArgc()));
783 writer
.returnFromIC();
785 return replaceICStub(entry
, fallback
, writer
, CacheKind::Call
);
788 bool TrialInliner::maybeInlineGetter(ICEntry
& entry
, ICFallbackStub
* fallback
,
789 BytecodeLocation loc
, CacheKind kind
) {
790 ICCacheIRStub
* stub
= maybeSingleStub(entry
);
795 MOZ_ASSERT(!icScript_
->hasInlinedChild(fallback
->pcOffset()));
797 Maybe
<InlinableGetterData
> data
= FindInlinableGetterData(stub
);
798 if (data
.isNothing()) {
802 MOZ_ASSERT(!data
->icScript
);
804 TrialInliningDecision inlining
= getInliningDecision(data
->target
, stub
, loc
);
805 // Decide whether to inline the target.
806 if (inlining
== TrialInliningDecision::NoInline
) {
810 if (inlining
== TrialInliningDecision::MonomorphicInline
) {
811 fallback
->setTrialInliningState(TrialInliningState::MonomorphicInlined
);
815 ICScript
* newICScript
= createInlinedICScript(data
->target
, loc
);
820 CacheIRWriter
writer(cx());
821 ValOperandId
valId(writer
.setInputOperandId(0));
822 if (kind
== CacheKind::GetElem
) {
823 // Register the key operand.
824 writer
.setInputOperandId(1);
826 cloneSharedPrefix(stub
, data
->endOfSharedPrefix
, writer
);
828 writer
.callInlinedGetterResult(data
->receiverOperand
, data
->target
,
829 newICScript
, data
->sameRealm
);
830 writer
.returnFromIC();
832 return replaceICStub(entry
, fallback
, writer
, kind
);
835 bool TrialInliner::maybeInlineSetter(ICEntry
& entry
, ICFallbackStub
* fallback
,
836 BytecodeLocation loc
, CacheKind kind
) {
837 ICCacheIRStub
* stub
= maybeSingleStub(entry
);
842 MOZ_ASSERT(!icScript_
->hasInlinedChild(fallback
->pcOffset()));
844 Maybe
<InlinableSetterData
> data
= FindInlinableSetterData(stub
);
845 if (data
.isNothing()) {
849 MOZ_ASSERT(!data
->icScript
);
851 TrialInliningDecision inlining
= getInliningDecision(data
->target
, stub
, loc
);
852 // Decide whether to inline the target.
853 if (inlining
== TrialInliningDecision::NoInline
) {
857 if (inlining
== TrialInliningDecision::MonomorphicInline
) {
858 fallback
->setTrialInliningState(TrialInliningState::MonomorphicInlined
);
862 ICScript
* newICScript
= createInlinedICScript(data
->target
, loc
);
867 CacheIRWriter
writer(cx());
868 ValOperandId
objValId(writer
.setInputOperandId(0));
869 ValOperandId
rhsValId(writer
.setInputOperandId(1));
870 cloneSharedPrefix(stub
, data
->endOfSharedPrefix
, writer
);
872 writer
.callInlinedSetter(data
->receiverOperand
, data
->target
,
873 data
->rhsOperand
, newICScript
, data
->sameRealm
);
874 writer
.returnFromIC();
876 return replaceICStub(entry
, fallback
, writer
, kind
);
879 bool TrialInliner::tryInlining() {
880 uint32_t numICEntries
= icScript_
->numICEntries();
881 BytecodeLocation startLoc
= script_
->location();
883 for (uint32_t icIndex
= 0; icIndex
< numICEntries
; icIndex
++) {
884 ICEntry
& entry
= icScript_
->icEntry(icIndex
);
885 ICFallbackStub
* fallback
= icScript_
->fallbackStub(icIndex
);
887 if (!TryFoldingStubs(cx(), fallback
, script_
, icScript_
)) {
891 BytecodeLocation loc
=
892 startLoc
+ BytecodeLocationOffset(fallback
->pcOffset());
893 JSOp op
= loc
.getOp();
896 case JSOp::CallContent
:
897 case JSOp::CallIgnoresRv
:
899 case JSOp::CallContentIter
:
901 case JSOp::NewContent
:
902 case JSOp::SuperCall
:
903 if (!maybeInlineCall(entry
, fallback
, loc
)) {
908 if (!maybeInlineGetter(entry
, fallback
, loc
, CacheKind::GetProp
)) {
913 if (!maybeInlineGetter(entry
, fallback
, loc
, CacheKind::GetElem
)) {
918 case JSOp::StrictSetProp
:
919 if (!maybeInlineSetter(entry
, fallback
, loc
, CacheKind::SetProp
)) {
931 InliningRoot
* TrialInliner::maybeGetInliningRoot() const {
932 if (auto* root
= icScript_
->inliningRoot()) {
936 MOZ_ASSERT(!icScript_
->isInlined());
937 return script_
->jitScript()->inliningRoot();
940 InliningRoot
* TrialInliner::getOrCreateInliningRoot() {
941 if (auto* root
= maybeGetInliningRoot()) {
944 return script_
->jitScript()->getOrCreateInliningRoot(cx(), script_
);
947 size_t TrialInliner::inliningRootTotalBytecodeSize() const {
948 if (auto* root
= maybeGetInliningRoot()) {
949 return root
->totalBytecodeSize();
951 return script_
->length();
954 bool InliningRoot::addInlinedScript(UniquePtr
<ICScript
> icScript
) {
955 return inlinedScripts_
.append(std::move(icScript
));
958 void InliningRoot::trace(JSTracer
* trc
) {
959 TraceEdge(trc
, &owningScript_
, "inlining-root-owning-script");
960 for (auto& inlinedScript
: inlinedScripts_
) {
961 inlinedScript
->trace(trc
);
965 bool InliningRoot::traceWeak(JSTracer
* trc
) {
966 bool allSurvived
= true;
967 for (auto& inlinedScript
: inlinedScripts_
) {
968 if (!inlinedScript
->traceWeak(trc
)) {
975 void InliningRoot::purgeInactiveICScripts() {
976 mozilla::DebugOnly
<uint32_t> totalSize
= owningScript_
->length();
978 for (auto& inlinedScript
: inlinedScripts_
) {
979 if (inlinedScript
->active()) {
980 totalSize
+= inlinedScript
->bytecodeSize();
982 MOZ_ASSERT(inlinedScript
->bytecodeSize() < totalBytecodeSize_
);
983 totalBytecodeSize_
-= inlinedScript
->bytecodeSize();
987 MOZ_ASSERT(totalBytecodeSize_
== totalSize
);
989 Zone
* zone
= owningScript_
->zone();
991 inlinedScripts_
.eraseIf([zone
](auto& inlinedScript
) {
992 if (inlinedScript
->active()) {
995 inlinedScript
->prepareForDestruction(zone
);